diff options
879 files changed, 30103 insertions, 13456 deletions
diff --git a/Documentation/ABI/testing/sysfs-platform-asus-wmi b/Documentation/ABI/testing/sysfs-platform-asus-wmi index 87ae5cc983bf..9e99f2909612 100644 --- a/Documentation/ABI/testing/sysfs-platform-asus-wmi +++ b/Documentation/ABI/testing/sysfs-platform-asus-wmi @@ -37,9 +37,9 @@ Contact: "AceLan Kao" <acelan.kao@canonical.com> Description: Resume on lid open. 1 means on, 0 means off. -What: /sys/devices/platform/<platform>/fan_mode -Date: Apr 2019 -KernelVersion: 5.2 +What: /sys/devices/platform/<platform>/fan_boost_mode +Date: Sep 2019 +KernelVersion: 5.3 Contact: "Yurii Pavlovskyi" <yurii.pavlovskyi@gmail.com> Description: Fan boost mode: diff --git a/Documentation/admin-guide/index.rst b/Documentation/admin-guide/index.rst index 280355d08af5..33feab2f4084 100644 --- a/Documentation/admin-guide/index.rst +++ b/Documentation/admin-guide/index.rst @@ -77,6 +77,7 @@ configure specific aspects of kernel behavior to your liking. blockdev/index ext4 binderfs + xfs pm/index thunderbolt LSM/index diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 34a363f91b46..f0461456d910 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2011,6 +2011,19 @@ Built with CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF=y, the default is off. + kprobe_event=[probe-list] + [FTRACE] Add kprobe events and enable at boot time. + The probe-list is a semicolon delimited list of probe + definitions. Each definition is same as kprobe_events + interface, but the parameters are comma delimited. + For example, to add a kprobe event on vfs_read with + arg1 and arg2, add to the command line; + + kprobe_event=p,vfs_read,$arg1,$arg2 + + See also Documentation/trace/kprobetrace.rst "Kernel + Boot Parameter" section. + kpti= [ARM64] Control page table isolation of user and kernel address spaces. Default: enabled on cores which need mitigation. @@ -2877,6 +2890,17 @@ /sys/module/printk/parameters/console_suspend) to turn on/off it dynamically. + novmcoredd [KNL,KDUMP] + Disable device dump. Device dump allows drivers to + append dump data to vmcore so you can collect driver + specified debug info. Drivers can append the data + without any limit and this data is stored in memory, + so this may cause significant memory stress. Disabling + device dump can help save memory but the driver debug + data will be no longer available. This parameter + is only available when CONFIG_PROC_VMCORE_DEVICE_DUMP + is set. + noaliencache [MM, NUMA, SLAB] Disables the allocation of alien caches in the slab allocator. Saves per-node memory, but will impact performance. diff --git a/Documentation/filesystems/xfs.txt b/Documentation/admin-guide/xfs.rst index a5cbb5e0e3db..e76665a8f2f2 100644 --- a/Documentation/filesystems/xfs.txt +++ b/Documentation/admin-guide/xfs.rst @@ -1,4 +1,6 @@ +.. SPDX-License-Identifier: GPL-2.0 +====================== The SGI XFS Filesystem ====================== @@ -18,8 +20,6 @@ Mount Options ============= When mounting an XFS filesystem, the following options are accepted. -For boolean mount options, the names with the (*) suffix is the -default behaviour. allocsize=size Sets the buffered I/O end-of-file preallocation size when @@ -31,46 +31,43 @@ default behaviour. preallocation size, which uses a set of heuristics to optimise the preallocation size based on the current allocation patterns within the file and the access patterns - to the file. Specifying a fixed allocsize value turns off + to the file. Specifying a fixed ``allocsize`` value turns off the dynamic behaviour. - attr2 - noattr2 + attr2 or noattr2 The options enable/disable an "opportunistic" improvement to be made in the way inline extended attributes are stored on-disk. When the new form is used for the first time when - attr2 is selected (either when setting or removing extended + ``attr2`` is selected (either when setting or removing extended attributes) the on-disk superblock feature bit field will be updated to reflect this format being in use. The default behaviour is determined by the on-disk feature - bit indicating that attr2 behaviour is active. If either - mount option it set, then that becomes the new default used + bit indicating that ``attr2`` behaviour is active. If either + mount option is set, then that becomes the new default used by the filesystem. - CRC enabled filesystems always use the attr2 format, and so - will reject the noattr2 mount option if it is set. + CRC enabled filesystems always use the ``attr2`` format, and so + will reject the ``noattr2`` mount option if it is set. - discard - nodiscard (*) + discard or nodiscard (default) Enable/disable the issuing of commands to let the block device reclaim space freed by the filesystem. This is useful for SSD devices, thinly provisioned LUNs and virtual machine images, but may have a performance impact. - Note: It is currently recommended that you use the fstrim - application to discard unused blocks rather than the discard + Note: It is currently recommended that you use the ``fstrim`` + application to ``discard`` unused blocks rather than the ``discard`` mount option because the performance impact of this option is quite severe. - grpid/bsdgroups - nogrpid/sysvgroups (*) + grpid/bsdgroups or nogrpid/sysvgroups (default) These options define what group ID a newly created file - gets. When grpid is set, it takes the group ID of the + gets. When ``grpid`` is set, it takes the group ID of the directory in which it is created; otherwise it takes the - fsgid of the current process, unless the directory has the - setgid bit set, in which case it takes the gid from the - parent directory, and also gets the setgid bit set if it is + ``fsgid`` of the current process, unless the directory has the + ``setgid`` bit set, in which case it takes the ``gid`` from the + parent directory, and also gets the ``setgid`` bit set if it is a directory itself. filestreams @@ -78,46 +75,42 @@ default behaviour. across the entire filesystem rather than just on directories configured to use it. - ikeep - noikeep (*) - When ikeep is specified, XFS does not delete empty inode - clusters and keeps them around on disk. When noikeep is + ikeep or noikeep (default) + When ``ikeep`` is specified, XFS does not delete empty inode + clusters and keeps them around on disk. When ``noikeep`` is specified, empty inode clusters are returned to the free space pool. - inode32 - inode64 (*) - When inode32 is specified, it indicates that XFS limits + inode32 or inode64 (default) + When ``inode32`` is specified, it indicates that XFS limits inode creation to locations which will not result in inode numbers with more than 32 bits of significance. - When inode64 is specified, it indicates that XFS is allowed + When ``inode64`` is specified, it indicates that XFS is allowed to create inodes at any location in the filesystem, including those which will result in inode numbers occupying - more than 32 bits of significance. + more than 32 bits of significance. - inode32 is provided for backwards compatibility with older + ``inode32`` is provided for backwards compatibility with older systems and applications, since 64 bits inode numbers might cause problems for some applications that cannot handle large inode numbers. If applications are in use which do - not handle inode numbers bigger than 32 bits, the inode32 + not handle inode numbers bigger than 32 bits, the ``inode32`` option should be specified. - - largeio - nolargeio (*) - If "nolargeio" is specified, the optimal I/O reported in - st_blksize by stat(2) will be as small as possible to allow + largeio or nolargeio (default) + If ``nolargeio`` is specified, the optimal I/O reported in + ``st_blksize`` by **stat(2)** will be as small as possible to allow user applications to avoid inefficient read/modify/write I/O. This is typically the page size of the machine, as this is the granularity of the page cache. - If "largeio" specified, a filesystem that was created with a - "swidth" specified will return the "swidth" value (in bytes) - in st_blksize. If the filesystem does not have a "swidth" - specified but does specify an "allocsize" then "allocsize" + If ``largeio`` is specified, a filesystem that was created with a + ``swidth`` specified will return the ``swidth`` value (in bytes) + in ``st_blksize``. If the filesystem does not have a ``swidth`` + specified but does specify an ``allocsize`` then ``allocsize`` (in bytes) will be returned instead. Otherwise the behaviour - is the same as if "nolargeio" was specified. + is the same as if ``nolargeio`` was specified. logbufs=value Set the number of in-memory log buffers. Valid numbers @@ -127,7 +120,7 @@ default behaviour. If the memory cost of 8 log buffers is too high on small systems, then it may be reduced at some cost to performance - on metadata intensive workloads. The logbsize option below + on metadata intensive workloads. The ``logbsize`` option below controls the size of each buffer and so is also relevant to this case. @@ -138,7 +131,7 @@ default behaviour. and 32768 (32k). Valid sizes for version 2 logs also include 65536 (64k), 131072 (128k) and 262144 (256k). The logbsize must be an integer multiple of the log - stripe unit configured at mkfs time. + stripe unit configured at **mkfs(8)** time. The default value for for version 1 logs is 32768, while the default value for version 2 logs is MAX(32768, log_sunit). @@ -153,21 +146,21 @@ default behaviour. noalign Data allocations will not be aligned at stripe unit boundaries. This is only relevant to filesystems created - with non-zero data alignment parameters (sunit, swidth) by - mkfs. + with non-zero data alignment parameters (``sunit``, ``swidth``) by + **mkfs(8)**. norecovery The filesystem will be mounted without running log recovery. If the filesystem was not cleanly unmounted, it is likely to - be inconsistent when mounted in "norecovery" mode. + be inconsistent when mounted in ``norecovery`` mode. Some files or directories may not be accessible because of this. - Filesystems mounted "norecovery" must be mounted read-only or + Filesystems mounted ``norecovery`` must be mounted read-only or the mount will fail. nouuid Don't check for double mounted file systems using the file - system uuid. This is useful to mount LVM snapshot volumes, - and often used in combination with "norecovery" for mounting + system ``uuid``. This is useful to mount LVM snapshot volumes, + and often used in combination with ``norecovery`` for mounting read-only snapshots. noquota @@ -176,15 +169,15 @@ default behaviour. uquota/usrquota/uqnoenforce/quota User disk quota accounting enabled, and limits (optionally) - enforced. Refer to xfs_quota(8) for further details. + enforced. Refer to **xfs_quota(8)** for further details. gquota/grpquota/gqnoenforce Group disk quota accounting enabled and limits (optionally) - enforced. Refer to xfs_quota(8) for further details. + enforced. Refer to **xfs_quota(8)** for further details. pquota/prjquota/pqnoenforce Project disk quota accounting enabled and limits (optionally) - enforced. Refer to xfs_quota(8) for further details. + enforced. Refer to **xfs_quota(8)** for further details. sunit=value and swidth=value Used to specify the stripe unit and width for a RAID device @@ -192,11 +185,11 @@ default behaviour. block units. These options are only relevant to filesystems that were created with non-zero data alignment parameters. - The sunit and swidth parameters specified must be compatible + The ``sunit`` and ``swidth`` parameters specified must be compatible with the existing filesystem alignment characteristics. In - general, that means the only valid changes to sunit are - increasing it by a power-of-2 multiple. Valid swidth values - are any integer multiple of a valid sunit value. + general, that means the only valid changes to ``sunit`` are + increasing it by a power-of-2 multiple. Valid ``swidth`` values + are any integer multiple of a valid ``sunit`` value. Typically the only time these mount options are necessary if after an underlying RAID device has had it's geometry @@ -221,22 +214,25 @@ default behaviour. Deprecated Mount Options ======================== +=========================== ================ Name Removal Schedule - ---- ---------------- +=========================== ================ +=========================== ================ Removed Mount Options ===================== +=========================== ======= Name Removed - ---- ------- +=========================== ======= delaylog/nodelaylog v4.0 ihashsize v4.0 irixsgid v4.0 osyncisdsync/osyncisosync v4.0 barrier v4.19 nobarrier v4.19 - +=========================== ======= sysctls ======= @@ -302,27 +298,27 @@ The following sysctls are available for the XFS filesystem: fs.xfs.inherit_sync (Min: 0 Default: 1 Max: 1) Setting this to "1" will cause the "sync" flag set - by the xfs_io(8) chattr command on a directory to be + by the **xfs_io(8)** chattr command on a directory to be inherited by files in that directory. fs.xfs.inherit_nodump (Min: 0 Default: 1 Max: 1) Setting this to "1" will cause the "nodump" flag set - by the xfs_io(8) chattr command on a directory to be + by the **xfs_io(8)** chattr command on a directory to be inherited by files in that directory. fs.xfs.inherit_noatime (Min: 0 Default: 1 Max: 1) Setting this to "1" will cause the "noatime" flag set - by the xfs_io(8) chattr command on a directory to be + by the **xfs_io(8)** chattr command on a directory to be inherited by files in that directory. fs.xfs.inherit_nosymlinks (Min: 0 Default: 1 Max: 1) Setting this to "1" will cause the "nosymlinks" flag set - by the xfs_io(8) chattr command on a directory to be + by the **xfs_io(8)** chattr command on a directory to be inherited by files in that directory. fs.xfs.inherit_nodefrag (Min: 0 Default: 1 Max: 1) Setting this to "1" will cause the "nodefrag" flag set - by the xfs_io(8) chattr command on a directory to be + by the **xfs_io(8)** chattr command on a directory to be inherited by files in that directory. fs.xfs.rotorstep (Min: 1 Default: 1 Max: 256) @@ -368,7 +364,7 @@ handler: -error handlers: Defines the behavior for a specific error. -The filesystem behavior during an error can be set via sysfs files. Each +The filesystem behavior during an error can be set via ``sysfs`` files. Each error handler works independently - the first condition met by an error handler for a specific class will cause the error to be propagated rather than reset and retried. @@ -419,7 +415,7 @@ level directory: handler configurations. Note: there is no guarantee that fail_at_unmount can be set while an - unmount is in progress. It is possible that the sysfs entries are + unmount is in progress. It is possible that the ``sysfs`` entries are removed by the unmounting filesystem before a "retry forever" error handler configuration causes unmount to hang, and hence the filesystem must be configured appropriately before unmount begins to prevent @@ -428,7 +424,7 @@ level directory: Each filesystem has specific error class handlers that define the error propagation behaviour for specific errors. There is also a "default" error handler defined, which defines the behaviour for all errors that don't have -specific handlers defined. Where multiple retry constraints are configuredi for +specific handlers defined. Where multiple retry constraints are configured for a single error, the first retry configuration that expires will cause the error to be propagated. The handler configurations are found in the directory: @@ -463,7 +459,7 @@ to be propagated. The handler configurations are found in the directory: Setting the value to "N" (where 0 < N < Max) will allow XFS to retry the operation for up to "N" seconds before propagating the error. -Note: The default behaviour for a specific error handler is dependent on both +**Note:** The default behaviour for a specific error handler is dependent on both the class and error context. For example, the default values for "metadata/ENODEV" are "0" rather than "-1" so that this error handler defaults to "fail immediately" behaviour. This is done because ENODEV is a fatal, diff --git a/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt b/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt index f378922906f6..a575e42f7fec 100644 --- a/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt +++ b/Documentation/devicetree/bindings/arm/freescale/fsl,scu.txt @@ -145,6 +145,16 @@ Optional Child nodes: - Data cells of ocotp: Detailed bindings are described in bindings/nvmem/nvmem.txt +Watchdog bindings based on SCU Message Protocol +------------------------------------------------------------ + +Required properties: +- compatible: should be: + "fsl,imx8qxp-sc-wdt" + followed by "fsl,imx-sc-wdt"; +Optional properties: +- timeout-sec: contains the watchdog timeout in seconds. + Example (imx8qxp): ------------- aliases { @@ -207,6 +217,11 @@ firmware { rtc: rtc { compatible = "fsl,imx8qxp-sc-rtc"; }; + + watchdog { + compatible = "fsl,imx8qxp-sc-wdt", "fsl,imx-sc-wdt"; + timeout-sec = <60>; + }; }; }; diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt index f3cef1a6d95c..07c9d813465c 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt @@ -10,6 +10,7 @@ Required Properties: - "mediatek,mt7622-audsys", "syscon" - "mediatek,mt7623-audsys", "mediatek,mt2701-audsys", "syscon" - "mediatek,mt8183-audiosys", "syscon" + - "mediatek,mt8516-audsys", "syscon" - #clock-cells: Must be 1 The AUDSYS controller uses the common clk binding from diff --git a/Documentation/devicetree/bindings/arm/stm32/mlahb.txt b/Documentation/devicetree/bindings/arm/stm32/mlahb.txt new file mode 100644 index 000000000000..25307aa1eb9b --- /dev/null +++ b/Documentation/devicetree/bindings/arm/stm32/mlahb.txt @@ -0,0 +1,37 @@ +ML-AHB interconnect bindings + +These bindings describe the STM32 SoCs ML-AHB interconnect bus which connects +a Cortex-M subsystem with dedicated memories. +The MCU SRAM and RETRAM memory parts can be accessed through different addresses +(see "RAM aliases" in [1]) using different buses (see [2]) : balancing the +Cortex-M firmware accesses among those ports allows to tune the system +performance. + +[1]: https://www.st.com/resource/en/reference_manual/dm00327659.pdf +[2]: https://wiki.st.com/stm32mpu/wiki/STM32MP15_RAM_mapping + +Required properties: +- compatible: should be "simple-bus" +- dma-ranges: describes memory addresses translation between the local CPU and + the remote Cortex-M processor. Each memory region, is declared with + 3 parameters: + - param 1: device base address (Cortex-M processor address) + - param 2: physical base address (local CPU address) + - param 3: size of the memory region. + +The Cortex-M remote processor accessed via the mlahb interconnect is described +by a child node. + +Example: +mlahb { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + dma-ranges = <0x00000000 0x38000000 0x10000>, + <0x10000000 0x10000000 0x60000>, + <0x30000000 0x30000000 0x60000>; + + m4_rproc: m4@10000000 { + ... + }; +}; diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml new file mode 100644 index 000000000000..c935405458fe --- /dev/null +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/allwinner,sun4i-a10-ccu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Allwinner Clock Control Unit Device Tree Bindings + +maintainers: + - Chen-Yu Tsai <wens@csie.org> + - Maxime Ripard <maxime.ripard@bootlin.com> + +properties: + "#clock-cells": + const: 1 + + "#reset-cells": + const: 1 + + compatible: + enum: + - allwinner,sun4i-a10-ccu + - allwinner,sun5i-a10s-ccu + - allwinner,sun5i-a13-ccu + - allwinner,sun6i-a31-ccu + - allwinner,sun7i-a20-ccu + - allwinner,sun8i-a23-ccu + - allwinner,sun8i-a33-ccu + - allwinner,sun8i-a83t-ccu + - allwinner,sun8i-a83t-r-ccu + - allwinner,sun8i-h3-ccu + - allwinner,sun8i-h3-r-ccu + - allwinner,sun8i-r40-ccu + - allwinner,sun8i-v3s-ccu + - allwinner,sun9i-a80-ccu + - allwinner,sun50i-a64-ccu + - allwinner,sun50i-a64-r-ccu + - allwinner,sun50i-h5-ccu + - allwinner,sun50i-h6-ccu + - allwinner,sun50i-h6-r-ccu + - allwinner,suniv-f1c100s-ccu + - nextthing,gr8-ccu + + reg: + maxItems: 1 + + clocks: + minItems: 2 + maxItems: 4 + items: + - description: High Frequency Oscillator (usually at 24MHz) + - description: Low Frequency Oscillator (usually at 32kHz) + - description: Internal Oscillator + - description: Peripherals PLL + + clock-names: + minItems: 2 + maxItems: 4 + items: + - const: hosc + - const: losc + - const: iosc + - const: pll-periph + +required: + - "#clock-cells" + - "#reset-cells" + - compatible + - reg + - clocks + - clock-names + +if: + properties: + compatible: + enum: + - allwinner,sun8i-a83t-r-ccu + - allwinner,sun8i-h3-r-ccu + - allwinner,sun50i-a64-r-ccu + - allwinner,sun50i-h6-r-ccu + +then: + properties: + clocks: + minItems: 4 + maxItems: 4 + + clock-names: + minItems: 4 + maxItems: 4 + +else: + if: + properties: + compatible: + const: allwinner,sun50i-h6-ccu + + then: + properties: + clocks: + minItems: 3 + maxItems: 3 + + clock-names: + minItems: 3 + maxItems: 3 + + else: + properties: + clocks: + minItems: 2 + maxItems: 2 + + clock-names: + minItems: 2 + maxItems: 2 + +additionalProperties: false + +examples: + - | + ccu: clock@1c20000 { + compatible = "allwinner,sun8i-h3-ccu"; + reg = <0x01c20000 0x400>; + clocks = <&osc24M>, <&osc32k>; + clock-names = "hosc", "losc"; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + - | + r_ccu: clock@1f01400 { + compatible = "allwinner,sun50i-a64-r-ccu"; + reg = <0x01f01400 0x100>; + clocks = <&osc24M>, <&osc32k>, <&iosc>, <&ccu 11>; + clock-names = "hosc", "losc", "iosc", "pll-periph"; + #clock-cells = <1>; + #reset-cells = <1>; + }; + +... diff --git a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt index 5c8b105be4d6..6eaa52092313 100644 --- a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt +++ b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt @@ -10,6 +10,7 @@ Required Properties: "amlogic,gxl-clkc" for GXL and GXM SoC, "amlogic,axg-clkc" for AXG SoC. "amlogic,g12a-clkc" for G12A SoC. + "amlogic,g12b-clkc" for G12B SoC. - clocks : list of clock phandle, one for each entry clock-names. - clock-names : should contain the following: * "xtal": the platform xtal diff --git a/Documentation/devicetree/bindings/clock/at91-clock.txt b/Documentation/devicetree/bindings/clock/at91-clock.txt index b520280e33ff..13f45db3b66d 100644 --- a/Documentation/devicetree/bindings/clock/at91-clock.txt +++ b/Documentation/devicetree/bindings/clock/at91-clock.txt @@ -9,10 +9,11 @@ Slow Clock controller: Required properties: - compatible : shall be one of the following: "atmel,at91sam9x5-sckc", - "atmel,sama5d3-sckc" or - "atmel,sama5d4-sckc": + "atmel,sama5d3-sckc", + "atmel,sama5d4-sckc" or + "microchip,sam9x60-sckc": at91 SCKC (Slow Clock Controller) -- #clock-cells : shall be 0. +- #clock-cells : shall be 1 for "microchip,sam9x60-sckc" otherwise shall be 0. - clocks : shall be the input parent clock phandle for the clock. Optional properties: diff --git a/Documentation/devicetree/bindings/clock/brcm,bcm63xx-clocks.txt b/Documentation/devicetree/bindings/clock/brcm,bcm63xx-clocks.txt new file mode 100644 index 000000000000..3041657e2f96 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/brcm,bcm63xx-clocks.txt @@ -0,0 +1,22 @@ +Gated Clock Controller Bindings for MIPS based BCM63XX SoCs + +Required properties: +- compatible: must be one of: + "brcm,bcm3368-clocks" + "brcm,bcm6328-clocks" + "brcm,bcm6358-clocks" + "brcm,bcm6362-clocks" + "brcm,bcm6368-clocks" + "brcm,bcm63268-clocks" + +- reg: Address and length of the register set +- #clock-cells: must be <1> + + +Example: + +clkctl: clock-controller@10000004 { + compatible = "brcm,bcm6328-clocks"; + reg = <0x10000004 0x4>; + #clock-cells = <1>; +}; diff --git a/Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt b/Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt index b8d8ef3bdc5f..52a064c789ee 100644 --- a/Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt +++ b/Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt @@ -40,6 +40,7 @@ Optional properties: input audio clocks from host system. - ln-psia1-mclk, ln-psia2-mclk : Optional input audio clocks from external connector. + - ln-spdif-mclk : Optional input audio clock from SPDIF. - ln-spdif-clkout : Optional input audio clock from SPDIF. - ln-adat-mclk : Optional input audio clock from ADAT. - ln-pmic-32k : On board fixed clock. diff --git a/Documentation/devicetree/bindings/clock/mvebu-core-clock.txt b/Documentation/devicetree/bindings/clock/mvebu-core-clock.txt index 796c260c183d..d8f5c490f893 100644 --- a/Documentation/devicetree/bindings/clock/mvebu-core-clock.txt +++ b/Documentation/devicetree/bindings/clock/mvebu-core-clock.txt @@ -59,6 +59,7 @@ Required properties: "marvell,dove-core-clock" - for Dove SoC core clocks "marvell,kirkwood-core-clock" - for Kirkwood SoC (except mv88f6180) "marvell,mv88f6180-core-clock" - for Kirkwood MV88f6180 SoC + "marvell,mv98dx1135-core-clock" - for Kirkwood 98dx1135 SoC "marvell,mv88f5181-core-clock" - for Orion MV88F5181 SoC "marvell,mv88f5182-core-clock" - for Orion MV88F5182 SoC "marvell,mv88f5281-core-clock" - for Orion MV88F5281 SoC diff --git a/Documentation/devicetree/bindings/clock/qcom,gpucc.txt b/Documentation/devicetree/bindings/clock/qcom,gpucc.txt index 4e5215ef1acd..269afe8a757e 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gpucc.txt +++ b/Documentation/devicetree/bindings/clock/qcom,gpucc.txt @@ -2,13 +2,15 @@ Qualcomm Graphics Clock & Reset Controller Binding -------------------------------------------------- Required properties : -- compatible : shall contain "qcom,sdm845-gpucc" +- compatible : shall contain "qcom,sdm845-gpucc" or "qcom,msm8998-gpucc" - reg : shall contain base register location and length - #clock-cells : from common clock binding, shall contain 1 - #reset-cells : from common reset binding, shall contain 1 - #power-domain-cells : from generic power domain binding, shall contain 1 - clocks : shall contain the XO clock + shall contain the gpll0 out main clock (msm8998) - clock-names : shall be "xo" + shall be "gpll0" (msm8998) Example: gpucc: clock-controller@5090000 { diff --git a/Documentation/devicetree/bindings/clock/renesas,r9a06g032-sysctrl.txt b/Documentation/devicetree/bindings/clock/renesas,r9a06g032-sysctrl.txt index d60b99756bb9..aed713cf0831 100644 --- a/Documentation/devicetree/bindings/clock/renesas,r9a06g032-sysctrl.txt +++ b/Documentation/devicetree/bindings/clock/renesas,r9a06g032-sysctrl.txt @@ -13,6 +13,7 @@ Required Properties: - external (optional) RGMII_REFCLK - clock-names: Must be: clock-names = "mclk", "rtc", "jtag", "rgmii_ref_ext"; + - #power-domain-cells: Must be 0 Examples -------- @@ -27,6 +28,7 @@ Examples clocks = <&ext_mclk>, <&ext_rtc_clk>, <&ext_jtag_clk>, <&ext_rgmii_ref>; clock-names = "mclk", "rtc", "jtag", "rgmii_ref_ext"; + #power-domain-cells = <0>; }; - Other nodes can use the clocks provided by SYSCTRL as in: @@ -38,6 +40,7 @@ Examples interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>; reg-shift = <2>; reg-io-width = <4>; - clocks = <&sysctrl R9A06G032_CLK_UART0>; - clock-names = "baudclk"; + clocks = <&sysctrl R9A06G032_CLK_UART0>, <&sysctrl R9A06G032_HCLK_UART0>; + clock-names = "baudclk", "apb_pclk"; + power-domains = <&sysctrl>; }; diff --git a/Documentation/devicetree/bindings/clock/silabs,si5341.txt b/Documentation/devicetree/bindings/clock/silabs,si5341.txt new file mode 100644 index 000000000000..a70c333e4cd4 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/silabs,si5341.txt @@ -0,0 +1,162 @@ +Binding for Silicon Labs Si5341 and Si5340 programmable i2c clock generator. + +Reference +[1] Si5341 Data Sheet + https://www.silabs.com/documents/public/data-sheets/Si5341-40-D-DataSheet.pdf +[2] Si5341 Reference Manual + https://www.silabs.com/documents/public/reference-manuals/Si5341-40-D-RM.pdf + +The Si5341 and Si5340 are programmable i2c clock generators with up to 10 output +clocks. The chip contains a PLL that sources 5 (or 4) multisynth clocks, which +in turn can be directed to any of the 10 (or 4) outputs through a divider. +The internal structure of the clock generators can be found in [2]. + +The driver can be used in "as is" mode, reading the current settings from the +chip at boot, in case you have a (pre-)programmed device. If the PLL is not +configured when the driver probes, it assumes the driver must fully initialize +it. + +The device type, speed grade and revision are determined runtime by probing. + +The driver currently only supports XTAL input mode, and does not support any +fancy input configurations. They can still be programmed into the chip and +the driver will leave them "as is". + +==I2C device node== + +Required properties: +- compatible: shall be one of the following: + "silabs,si5340" - Si5340 A/B/C/D + "silabs,si5341" - Si5341 A/B/C/D +- reg: i2c device address, usually 0x74 +- #clock-cells: from common clock binding; shall be set to 2. + The first value is "0" for outputs, "1" for synthesizers. + The second value is the output or synthesizer index. +- clocks: from common clock binding; list of parent clock handles, + corresponding to inputs. Use a fixed clock for the "xtal" input. + At least one must be present. +- clock-names: One of: "xtal", "in0", "in1", "in2" +- vdd-supply: Regulator node for VDD + +Optional properties: +- vdda-supply: Regulator node for VDDA +- vdds-supply: Regulator node for VDDS +- silabs,pll-m-num, silabs,pll-m-den: Numerator and denominator for PLL + feedback divider. Must be such that the PLL output is in the valid range. For + example, to create 14GHz from a 48MHz xtal, use m-num=14000 and m-den=48. Only + the fraction matters, using 3500 and 12 will deliver the exact same result. + If these are not specified, and the PLL is not yet programmed when the driver + probes, the PLL will be set to 14GHz. +- silabs,reprogram: When present, the driver will always assume the device must + be initialized, and always performs the soft-reset routine. Since this will + temporarily stop all output clocks, don't do this if the chip is generating + the CPU clock for example. +- interrupts: Interrupt for INTRb pin. +- #address-cells: shall be set to 1. +- #size-cells: shall be set to 0. + + +== Child nodes: Outputs == + +The child nodes list the output clocks. + +Each of the clock outputs can be overwritten individually by using a child node. +If a child node for a clock output is not set, the configuration remains +unchanged. + +Required child node properties: +- reg: number of clock output. + +Optional child node properties: +- vdd-supply: Regulator node for VDD for this output. The driver selects default + values for common-mode and amplitude based on the voltage. +- silabs,format: Output format, one of: + 1 = differential (defaults to LVDS levels) + 2 = low-power (defaults to HCSL levels) + 4 = LVCMOS +- silabs,common-mode: Manually override output common mode, see [2] for values +- silabs,amplitude: Manually override output amplitude, see [2] for values +- silabs,synth-master: boolean. If present, this output is allowed to change the + multisynth frequency dynamically. +- silabs,silabs,disable-high: boolean. If set, the clock output is driven HIGH + when disabled, otherwise it's driven LOW. + +==Example== + +/* 48MHz reference crystal */ +ref48: ref48M { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <48000000>; +}; + +i2c-master-node { + /* Programmable clock (for logic) */ + si5341: clock-generator@74 { + reg = <0x74>; + compatible = "silabs,si5341"; + #clock-cells = <2>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&ref48>; + clock-names = "xtal"; + + silabs,pll-m-num = <14000>; /* PLL at 14.0 GHz */ + silabs,pll-m-den = <48>; + silabs,reprogram; /* Chips are not programmed, always reset */ + + out@0 { + reg = <0>; + silabs,format = <1>; /* LVDS 3v3 */ + silabs,common-mode = <3>; + silabs,amplitude = <3>; + silabs,synth-master; + }; + + /* + * Output 6 configuration: + * LVDS 1v8 + */ + out@6 { + reg = <6>; + silabs,format = <1>; /* LVDS 1v8 */ + silabs,common-mode = <13>; + silabs,amplitude = <3>; + }; + + /* + * Output 8 configuration: + * HCSL 3v3 + */ + out@8 { + reg = <8>; + silabs,format = <2>; + silabs,common-mode = <11>; + silabs,amplitude = <3>; + }; + }; +}; + +some-video-node { + /* Standard clock bindings */ + clock-names = "pixel"; + clocks = <&si5341 0 7>; /* Output 7 */ + + /* Set output 7 to use syntesizer 3 as its parent */ + assigned-clocks = <&si5341 0 7>, <&si5341 1 3>; + assigned-clock-parents = <&si5341 1 3>; + /* Set output 7 to 148.5 MHz using a synth frequency of 594 MHz */ + assigned-clock-rates = <148500000>, <594000000>; +}; + +some-audio-node { + clock-names = "i2s-clk"; + clocks = <&si5341 0 0>; + /* + * since output 0 is a synth-master, the synth will be automatically set + * to an appropriate frequency when the audio driver requests another + * frequency. We give control over synth 2 to this output here. + */ + assigned-clocks = <&si5341 0 0>; + assigned-clock-parents = <&si5341 1 2>; +}; diff --git a/Documentation/devicetree/bindings/clock/sunxi-ccu.txt b/Documentation/devicetree/bindings/clock/sunxi-ccu.txt deleted file mode 100644 index e3bd88ae456b..000000000000 --- a/Documentation/devicetree/bindings/clock/sunxi-ccu.txt +++ /dev/null @@ -1,62 +0,0 @@ -Allwinner Clock Control Unit Binding ------------------------------------- - -Required properties : -- compatible: must contain one of the following compatibles: - - "allwinner,sun4i-a10-ccu" - - "allwinner,sun5i-a10s-ccu" - - "allwinner,sun5i-a13-ccu" - - "allwinner,sun6i-a31-ccu" - - "allwinner,sun7i-a20-ccu" - - "allwinner,sun8i-a23-ccu" - - "allwinner,sun8i-a33-ccu" - - "allwinner,sun8i-a83t-ccu" - - "allwinner,sun8i-a83t-r-ccu" - - "allwinner,sun8i-h3-ccu" - - "allwinner,sun8i-h3-r-ccu" -+ - "allwinner,sun8i-r40-ccu" - - "allwinner,sun8i-v3s-ccu" - - "allwinner,sun9i-a80-ccu" - - "allwinner,sun50i-a64-ccu" - - "allwinner,sun50i-a64-r-ccu" - - "allwinner,sun50i-h5-ccu" - - "allwinner,sun50i-h6-ccu" - - "allwinner,sun50i-h6-r-ccu" - - "allwinner,suniv-f1c100s-ccu" - - "nextthing,gr8-ccu" - -- reg: Must contain the registers base address and length -- clocks: phandle to the oscillators feeding the CCU. Two are needed: - - "hosc": the high frequency oscillator (usually at 24MHz) - - "losc": the low frequency oscillator (usually at 32kHz) - On the A83T, this is the internal 16MHz oscillator divided by 512 -- clock-names: Must contain the clock names described just above -- #clock-cells : must contain 1 -- #reset-cells : must contain 1 - -For the main CCU on H6, one more clock is needed: -- "iosc": the SoC's internal frequency oscillator - -For the PRCM CCUs on A83T/H3/A64/H6, two more clocks are needed: -- "pll-periph": the SoC's peripheral PLL from the main CCU -- "iosc": the SoC's internal frequency oscillator - -Example for generic CCU: -ccu: clock@1c20000 { - compatible = "allwinner,sun8i-h3-ccu"; - reg = <0x01c20000 0x400>; - clocks = <&osc24M>, <&osc32k>; - clock-names = "hosc", "losc"; - #clock-cells = <1>; - #reset-cells = <1>; -}; - -Example for PRCM CCU: -r_ccu: clock@1f01400 { - compatible = "allwinner,sun50i-a64-r-ccu"; - reg = <0x01f01400 0x100>; - clocks = <&osc24M>, <&osc32k>, <&iosc>, <&ccu CLK_PLL_PERIPH0>; - clock-names = "hosc", "losc", "iosc", "pll-periph"; - #clock-cells = <1>; - #reset-cells = <1>; -}; diff --git a/Documentation/devicetree/bindings/dma/8250_mtk_dma.txt b/Documentation/devicetree/bindings/dma/8250_mtk_dma.txt deleted file mode 100644 index 3fe0961bcf64..000000000000 --- a/Documentation/devicetree/bindings/dma/8250_mtk_dma.txt +++ /dev/null @@ -1,33 +0,0 @@ -* Mediatek UART APDMA Controller - -Required properties: -- compatible should contain: - * "mediatek,mt2712-uart-dma" for MT2712 compatible APDMA - * "mediatek,mt6577-uart-dma" for MT6577 and all of the above - -- reg: The base address of the APDMA register bank. - -- interrupts: A single interrupt specifier. - -- clocks : Must contain an entry for each entry in clock-names. - See ../clocks/clock-bindings.txt for details. -- clock-names: The APDMA clock for register accesses - -Examples: - - apdma: dma-controller@11000380 { - compatible = "mediatek,mt2712-uart-dma"; - reg = <0 0x11000380 0 0x400>; - interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_LOW>, - <GIC_SPI 64 IRQ_TYPE_LEVEL_LOW>, - <GIC_SPI 65 IRQ_TYPE_LEVEL_LOW>, - <GIC_SPI 66 IRQ_TYPE_LEVEL_LOW>, - <GIC_SPI 67 IRQ_TYPE_LEVEL_LOW>, - <GIC_SPI 68 IRQ_TYPE_LEVEL_LOW>, - <GIC_SPI 69 IRQ_TYPE_LEVEL_LOW>, - <GIC_SPI 70 IRQ_TYPE_LEVEL_LOW>; - clocks = <&pericfg CLK_PERI_AP_DMA>; - clock-names = "apdma"; - #dma-cells = <1>; - }; - diff --git a/Documentation/devicetree/bindings/dma/arm-pl330.txt b/Documentation/devicetree/bindings/dma/arm-pl330.txt index db7e2260f9c5..2c7fd1941abb 100644 --- a/Documentation/devicetree/bindings/dma/arm-pl330.txt +++ b/Documentation/devicetree/bindings/dma/arm-pl330.txt @@ -16,6 +16,9 @@ Optional properties: - dma-channels: contains the total number of DMA channels supported by the DMAC - dma-requests: contains the total number of DMA requests supported by the DMAC - arm,pl330-broken-no-flushp: quirk for avoiding to execute DMAFLUSHP + - resets: contains an entry for each entry in reset-names. + See ../reset/reset.txt for details. + - reset-names: must contain at least "dma", and optional is "dma-ocp". Example: diff --git a/Documentation/devicetree/bindings/dma/fsl-edma.txt b/Documentation/devicetree/bindings/dma/fsl-edma.txt index 97e213e07660..29dd3ccb1235 100644 --- a/Documentation/devicetree/bindings/dma/fsl-edma.txt +++ b/Documentation/devicetree/bindings/dma/fsl-edma.txt @@ -9,15 +9,16 @@ group, DMAMUX0 or DMAMUX1, but not both. Required properties: - compatible : - "fsl,vf610-edma" for eDMA used similar to that on Vybrid vf610 SoC + - "fsl,imx7ulp-edma" for eDMA2 used similar to that on i.mx7ulp - reg : Specifies base physical address(s) and size of the eDMA registers. The 1st region is eDMA control register's address and size. The 2nd and the 3rd regions are programmable channel multiplexing control register's address and size. - interrupts : A list of interrupt-specifiers, one for each entry in - interrupt-names. -- interrupt-names : Should contain: - "edma-tx" - the transmission interrupt - "edma-err" - the error interrupt + interrupt-names on vf610 similar SoC. But for i.mx7ulp per channel + per transmission interrupt, total 16 channel interrupt and 1 + error interrupt(located in the last), no interrupt-names list on + i.mx7ulp for clean on dts. - #dma-cells : Must be <2>. The 1st cell specifies the DMAMUX(0 for DMAMUX0 and 1 for DMAMUX1). Specific request source can only be multiplexed by specific channels @@ -28,6 +29,7 @@ Required properties: - clock-names : A list of channel group clock names. Should contain: "dmamux0" - clock name of mux0 group "dmamux1" - clock name of mux1 group + Note: No dmamux0 on i.mx7ulp, but another 'dma' clk added on i.mx7ulp. - clocks : A list of phandle and clock-specifier pairs, one for each entry in clock-names. @@ -35,6 +37,10 @@ Optional properties: - big-endian: If present registers and hardware scatter/gather descriptors of the eDMA are implemented in big endian mode, otherwise in little mode. +- interrupt-names : Should contain the below on vf610 similar SoC but not used + on i.mx7ulp similar SoC: + "edma-tx" - the transmission interrupt + "edma-err" - the error interrupt Examples: @@ -52,8 +58,36 @@ edma0: dma-controller@40018000 { clock-names = "dmamux0", "dmamux1"; clocks = <&clks VF610_CLK_DMAMUX0>, <&clks VF610_CLK_DMAMUX1>; -}; +}; /* vf610 */ +edma1: dma-controller@40080000 { + #dma-cells = <2>; + compatible = "fsl,imx7ulp-edma"; + reg = <0x40080000 0x2000>, + <0x40210000 0x1000>; + dma-channels = <32>; + interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>, + /* last is eDMA2-ERR interrupt */ + <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>; + clock-names = "dma", "dmamux0"; + clocks = <&pcc2 IMX7ULP_CLK_DMA1>, + <&pcc2 IMX7ULP_CLK_DMA_MUX1>; +}; /* i.mx7ulp */ * DMA clients DMA client drivers that uses the DMA function must use the format described diff --git a/Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt b/Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt new file mode 100644 index 000000000000..5d6f98c43e3d --- /dev/null +++ b/Documentation/devicetree/bindings/dma/mtk-uart-apdma.txt @@ -0,0 +1,54 @@ +* Mediatek UART APDMA Controller + +Required properties: +- compatible should contain: + * "mediatek,mt2712-uart-dma" for MT2712 compatible APDMA + * "mediatek,mt6577-uart-dma" for MT6577 and all of the above + +- reg: The base address of the APDMA register bank. + +- interrupts: A single interrupt specifier. + One interrupt per dma-requests, or 8 if no dma-requests property is present + +- dma-requests: The number of DMA channels + +- clocks : Must contain an entry for each entry in clock-names. + See ../clocks/clock-bindings.txt for details. +- clock-names: The APDMA clock for register accesses + +- mediatek,dma-33bits: Present if the DMA requires support + +Examples: + + apdma: dma-controller@11000400 { + compatible = "mediatek,mt2712-uart-dma"; + reg = <0 0x11000400 0 0x80>, + <0 0x11000480 0 0x80>, + <0 0x11000500 0 0x80>, + <0 0x11000580 0 0x80>, + <0 0x11000600 0 0x80>, + <0 0x11000680 0 0x80>, + <0 0x11000700 0 0x80>, + <0 0x11000780 0 0x80>, + <0 0x11000800 0 0x80>, + <0 0x11000880 0 0x80>, + <0 0x11000900 0 0x80>, + <0 0x11000980 0 0x80>; + interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 105 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 106 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 107 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 108 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 109 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 110 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 111 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 112 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 113 IRQ_TYPE_LEVEL_LOW>, + <GIC_SPI 114 IRQ_TYPE_LEVEL_LOW>; + dma-requests = <12>; + clocks = <&pericfg CLK_PERI_AP_DMA>; + clock-names = "apdma"; + mediatek,dma-33bits; + #dma-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/dma/sun6i-dma.txt b/Documentation/devicetree/bindings/dma/sun6i-dma.txt index 7fccc20d8331..cae31f4e77ba 100644 --- a/Documentation/devicetree/bindings/dma/sun6i-dma.txt +++ b/Documentation/devicetree/bindings/dma/sun6i-dma.txt @@ -28,12 +28,17 @@ Example: }; ------------------------------------------------------------------------------ -For A64 DMA controller: +For A64 and H6 DMA controller: Required properties: -- compatible: "allwinner,sun50i-a64-dma" +- compatible: Must be one of + "allwinner,sun50i-a64-dma" + "allwinner,sun50i-h6-dma" - dma-channels: Number of DMA channels supported by the controller. Refer to Documentation/devicetree/bindings/dma/dma.txt +- clocks: In addition to parent AHB clock, it should also contain mbus + clock (H6 only) +- clock-names: Should contain "bus" and "mbus" (H6 only) - all properties above, i.e. reg, interrupts, clocks, resets and #dma-cells Optional properties: diff --git a/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt b/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt index 2c9804f4f4ac..8d365f89694c 100644 --- a/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt +++ b/Documentation/devicetree/bindings/hwlock/omap-hwspinlock.txt @@ -1,12 +1,16 @@ -OMAP4+ HwSpinlock Driver -======================== +TI HwSpinlock for OMAP and K3 based SoCs +========================================= Required properties: -- compatible: Should be "ti,omap4-hwspinlock" for - OMAP44xx, OMAP54xx, AM33xx, AM43xx, DRA7xx SoCs +- compatible: Should be one of the following, + "ti,omap4-hwspinlock" for + OMAP44xx, OMAP54xx, AM33xx, AM43xx, DRA7xx SoCs + "ti,am654-hwspinlock" for + K3 AM65x and J721E SoCs - reg: Contains the hwspinlock module register address space (base address and length) - ti,hwmods: Name of the hwmod associated with the hwspinlock device + (for OMAP architecture based SoCs only) - #hwlock-cells: Should be 1. The OMAP hwspinlock users will use a 0-indexed relative hwlock number as the argument specifier value for requesting a specific hwspinlock @@ -17,10 +21,21 @@ Please look at the generic hwlock binding for usage information for consumers, Example: -/* OMAP4 */ +1. OMAP4 SoCs hwspinlock: spinlock@4a0f6000 { compatible = "ti,omap4-hwspinlock"; reg = <0x4a0f6000 0x1000>; ti,hwmods = "spinlock"; #hwlock-cells = <1>; }; + +2. AM65x SoCs and J721E SoCs +&cbass_main { + cbass_main_navss: interconnect0 { + hwspinlock: spinlock@30e00000 { + compatible = "ti,am654-hwspinlock"; + reg = <0x00 0x30e00000 0x00 0x1000>; + #hwlock-cells = <1>; + }; + }; +}; diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp-pil.txt b/Documentation/devicetree/bindings/remoteproc/qcom,hexagon-v56.txt index 66af2c30944f..1337a3d93d35 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp-pil.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,hexagon-v56.txt @@ -1,12 +1,13 @@ -Qualcomm Technology Inc. ADSP Peripheral Image Loader +Qualcomm Technology Inc. Hexagon v56 Peripheral Image Loader This document defines the binding for a component that loads and boots firmware -on the Qualcomm Technology Inc. ADSP Hexagon core. +on the Qualcomm Technology Inc. Hexagon v56 core. - compatible: Usage: required Value type: <string> Definition: must be one of: + "qcom,qcs404-cdsp-pil", "qcom,sdm845-adsp-pil" - reg: @@ -28,10 +29,11 @@ on the Qualcomm Technology Inc. ADSP Hexagon core. - clocks: Usage: required Value type: <prop-encoded-array> - Definition: List of 8 phandle and clock specifier pairs for the adsp. + Definition: List of phandles and clock specifier pairs for the Hexagon, + per clock-names below. - clock-names: - Usage: required + Usage: required for SDM845 ADSP Value type: <stringlist> Definition: List of clock input name strings sorted in the same order as the clocks property. Definition must have @@ -39,6 +41,14 @@ on the Qualcomm Technology Inc. ADSP Hexagon core. "lpass_ahbm_aon_cbcr", "qdsp6ss_xo", "qdsp6ss_sleep" and "qdsp6ss_core". +- clock-names: + Usage: required for QCS404 CDSP + Value type: <stringlist> + Definition: List of clock input name strings sorted in the same + order as the clocks property. Definition must have + "xo", "sway", "tbu", "bimc", "ahb_aon", "q6ss_slave", + "q6ss_master", "q6_axim". + - power-domains: Usage: required Value type: <phandle> @@ -47,28 +57,33 @@ on the Qualcomm Technology Inc. ADSP Hexagon core. - resets: Usage: required Value type: <phandle> - Definition: reference to the list of 2 reset-controller for the adsp. + Definition: reference to the list of resets for the Hexagon. - reset-names: - Usage: required + Usage: required for SDM845 ADSP Value type: <stringlist> Definition: must be "pdc_sync" and "cc_lpass" +- reset-names: + Usage: required for QCS404 CDSP + Value type: <stringlist> + Definition: must be "restart" + - qcom,halt-regs: Usage: required Value type: <prop-encoded-array> Definition: a phandle reference to a syscon representing TCSR followed - by the offset within syscon for lpass halt register. + by the offset within syscon for Hexagon halt register. - memory-region: Usage: required Value type: <phandle> - Definition: reference to the reserved-memory for the ADSP + Definition: reference to the reserved-memory for the firmware - qcom,smem-states: Usage: required Value type: <phandle> - Definition: reference to the smem state for requesting the ADSP to + Definition: reference to the smem state for requesting the Hexagon to shut down - qcom,smem-state-names: @@ -79,7 +94,7 @@ on the Qualcomm Technology Inc. ADSP Hexagon core. = SUBNODES The adsp node may have an subnode named "glink-edge" that describes the -communication edge, channels and devices related to the ADSP. +communication edge, channels and devices related to the Hexagon. See ../soc/qcom/qcom,glink.txt for details on how to describe these. = EXAMPLE diff --git a/Documentation/devicetree/bindings/remoteproc/stm32-rproc.txt b/Documentation/devicetree/bindings/remoteproc/stm32-rproc.txt new file mode 100644 index 000000000000..5fa915a4b736 --- /dev/null +++ b/Documentation/devicetree/bindings/remoteproc/stm32-rproc.txt @@ -0,0 +1,63 @@ +STMicroelectronics STM32 Remoteproc +----------------------------------- +This document defines the binding for the remoteproc component that loads and +boots firmwares on the ST32MP family chipset. + +Required properties: +- compatible: Must be "st,stm32mp1-m4" +- reg: Address ranges of the RETRAM and MCU SRAM memories used by the + remote processor. +- resets: Reference to a reset controller asserting the remote processor. +- st,syscfg-holdboot: Reference to the system configuration which holds the + remote processor reset hold boot + 1st cell: phandle of syscon block + 2nd cell: register offset containing the hold boot setting + 3rd cell: register bitmask for the hold boot field +- st,syscfg-tz: Reference to the system configuration which holds the RCC trust + zone mode + 1st cell: phandle to syscon block + 2nd cell: register offset containing the RCC trust zone mode setting + 3rd cell: register bitmask for the RCC trust zone mode bit + +Optional properties: +- interrupts: Should contain the watchdog interrupt +- mboxes: This property is required only if the rpmsg/virtio functionality + is used. List of phandle and mailbox channel specifiers: + - a channel (a) used to communicate through virtqueues with the + remote proc. + Bi-directional channel: + - from local to remote = send message + - from remote to local = send message ack + - a channel (b) working the opposite direction of channel (a) + - a channel (c) used by the local proc to notify the remote proc + that it is about to be shut down. + Unidirectional channel: + - from local to remote, where ACK from the remote means + that it is ready for shutdown +- mbox-names: This property is required if the mboxes property is used. + - must be "vq0" for channel (a) + - must be "vq1" for channel (b) + - must be "shutdown" for channel (c) +- memory-region: List of phandles to the reserved memory regions associated with + the remoteproc device. This is variable and describes the + memories shared with the remote processor (eg: remoteproc + firmware and carveouts, rpmsg vrings, ...). + (see ../reserved-memory/reserved-memory.txt) +- st,syscfg-pdds: Reference to the system configuration which holds the remote + processor deep sleep setting + 1st cell: phandle to syscon block + 2nd cell: register offset containing the deep sleep setting + 3rd cell: register bitmask for the deep sleep bit +- st,auto-boot: If defined, when remoteproc is probed, it loads the default + firmware and starts the remote processor. + +Example: + m4_rproc: m4@10000000 { + compatible = "st,stm32mp1-m4"; + reg = <0x10000000 0x40000>, + <0x30000000 0x40000>, + <0x38000000 0x10000>; + resets = <&rcc MCU_R>; + st,syscfg-holdboot = <&rcc 0x10C 0x1>; + st,syscfg-tz = <&rcc 0x000 0x1>; + }; diff --git a/Documentation/devicetree/bindings/rtc/allwinner,sun4i-a10-rtc.yaml b/Documentation/devicetree/bindings/rtc/allwinner,sun4i-a10-rtc.yaml new file mode 100644 index 000000000000..46d69c32b89b --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/allwinner,sun4i-a10-rtc.yaml @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/allwinner,sun4i-a10-rtc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Allwinner A10 RTC Device Tree Bindings + +allOf: + - $ref: "rtc.yaml#" + +maintainers: + - Chen-Yu Tsai <wens@csie.org> + - Maxime Ripard <maxime.ripard@bootlin.com> + +properties: + compatible: + enum: + - allwinner,sun4i-a10-rtc + - allwinner,sun7i-a20-rtc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + rtc: rtc@1c20d00 { + compatible = "allwinner,sun4i-a10-rtc"; + reg = <0x01c20d00 0x20>; + interrupts = <24>; + }; + +... diff --git a/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml new file mode 100644 index 000000000000..924622f39c44 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/allwinner,sun6i-a31-rtc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Allwinner A31 RTC Device Tree Bindings + +maintainers: + - Chen-Yu Tsai <wens@csie.org> + - Maxime Ripard <maxime.ripard@bootlin.com> + +properties: + "#clock-cells": + const: 1 + + compatible: + oneOf: + - const: allwinner,sun6i-a31-rtc + - const: allwinner,sun8i-a23-rtc + - const: allwinner,sun8i-h3-rtc + - const: allwinner,sun8i-r40-rtc + - const: allwinner,sun8i-v3-rtc + - const: allwinner,sun50i-h5-rtc + - items: + - const: allwinner,sun50i-a64-rtc + - const: allwinner,sun8i-h3-rtc + + reg: + maxItems: 1 + + interrupts: + minItems: 1 + maxItems: 2 + items: + - description: RTC Alarm 0 + - description: RTC Alarm 1 + + clocks: + maxItems: 1 + + clock-output-names: + minItems: 1 + maxItems: 3 + description: + The RTC provides up to three clocks + - the Low Frequency Oscillator or LOSC, at index 0, + - the Low Frequency Oscillator External output (X32KFOUT in + the datasheet), at index 1, + - the Internal Oscillator, at index 2. + +allOf: + - $ref: "rtc.yaml#" + - if: + properties: + compatible: + contains: + const: allwinner,sun6i-a31-rtc + + then: + properties: + clock-output-names: + minItems: 1 + maxItems: 1 + + - if: + properties: + compatible: + contains: + enum: + - allwinner,sun8i-a23-rtc + - allwinner,sun8i-r40-rtc + - allwinner,sun8i-v3-rtc + + then: + properties: + clock-output-names: + minItems: 2 + maxItems: 2 + + - if: + properties: + compatible: + contains: + enum: + - allwinner,sun8i-h3-rtc + - allwinner,sun50i-h5-rtc + + then: + properties: + clock-output-names: + minItems: 3 + maxItems: 3 + + - if: + properties: + compatible: + contains: + const: allwinner,sun8i-r40-rtc + + then: + properties: + interrupts: + minItems: 1 + maxItems: 1 + + else: + properties: + interrupts: + minItems: 2 + maxItems: 2 + +required: + - "#clock-cells" + - compatible + - reg + - interrupts + - clocks + - clock-output-names + +additionalProperties: false + +examples: + - | + rtc: rtc@1f00000 { + compatible = "allwinner,sun6i-a31-rtc"; + reg = <0x01f00000 0x400>; + interrupts = <0 40 4>, <0 41 4>; + clock-output-names = "osc32k"; + clocks = <&ext_osc32k>; + #clock-cells = <1>; + }; + +... diff --git a/Documentation/devicetree/bindings/rtc/rtc.txt b/Documentation/devicetree/bindings/rtc/rtc.txt index a97fc6a9a75e..b8d36fce5e2d 100644 --- a/Documentation/devicetree/bindings/rtc/rtc.txt +++ b/Documentation/devicetree/bindings/rtc/rtc.txt @@ -1,72 +1 @@ -Generic device tree bindings for Real Time Clock devices -======================================================== - -This document describes generic bindings which can be used to describe Real Time -Clock devices in a device tree. - -Required properties -------------------- - -- compatible : name of RTC device following generic names recommended practice. - -For other required properties e.g. to describe register sets, -clocks, etc. check the binding documentation of the specific driver. - -Optional properties -------------------- - -- start-year : if provided, the default hardware range supported by the RTC is - shifted so the first usable year is the specified one. - -The following properties may not be supported by all drivers. However, if a -driver wants to support one of the below features, it should adapt the bindings -below. -- trickle-resistor-ohms : Selected resistor for trickle charger. Should be given - if trickle charger should be enabled -- trickle-diode-disable : Do not use internal trickle charger diode Should be - given if internal trickle charger diode should be - disabled -- wakeup-source : Enables wake up of host system on alarm -- quartz-load-femtofarads : The capacitive load of the quartz(x-tal), - expressed in femto Farad (fF). - The default value shall be listed (if optional), - and likewise all valid values. - -Trivial RTCs ------------- - -This is a list of trivial RTC devices that have simple device tree -bindings, consisting only of a compatible field, an address and -possibly an interrupt line. - - -Compatible Vendor / Chip -========== ============= -abracon,abb5zes3 AB-RTCMC-32.768kHz-B5ZE-S3: Real Time Clock/Calendar Module with I2C Interface -abracon,abeoz9 AB-RTCMC-32.768kHz-EOZ9: Real Time Clock/Calendar Module with I2C Interface -dallas,ds1374 I2C, 32-Bit Binary Counter Watchdog RTC with Trickle Charger and Reset Input/Output -dallas,ds1672 Dallas DS1672 Real-time Clock -dallas,ds3232 Extremely Accurate I²C RTC with Integrated Crystal and SRAM -epson,rx8010 I2C-BUS INTERFACE REAL TIME CLOCK MODULE -epson,rx8571 I2C-BUS INTERFACE REAL TIME CLOCK MODULE with Battery Backed RAM -epson,rx8581 I2C-BUS INTERFACE REAL TIME CLOCK MODULE -emmicro,em3027 EM Microelectronic EM3027 Real-time Clock -isil,isl1208 Intersil ISL1208 Low Power RTC with Battery Backed SRAM -isil,isl1218 Intersil ISL1218 Low Power RTC with Battery Backed SRAM -isil,isl12022 Intersil ISL12022 Real-time Clock -microcrystal,rv3028 Real Time Clock Module with I2C-Bus -microcrystal,rv3029 Real Time Clock Module with I2C-Bus -microcrystal,rv8523 Real Time Clock -nxp,pcf2127 Real-time clock -nxp,pcf2129 Real-time clock -nxp,pcf8563 Real-time clock/calendar -pericom,pt7c4338 Real-time Clock Module -ricoh,r2025sd I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC -ricoh,r2221tl I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC -ricoh,rs5c372a I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC -ricoh,rs5c372b I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC -ricoh,rv5c386 I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC -ricoh,rv5c387a I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC -sii,s35390a 2-wire CMOS real-time clock -whwave,sd3078 I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC -xircom,x1205 Xircom X1205 I2C RTC +This file has been moved to rtc.yaml. diff --git a/Documentation/devicetree/bindings/rtc/rtc.yaml b/Documentation/devicetree/bindings/rtc/rtc.yaml new file mode 100644 index 000000000000..ee237b2ed66a --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/rtc.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/rtc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: RTC Generic Binding + +maintainers: + - Alexandre Belloni <alexandre.belloni@bootlin.com> + +description: | + This document describes generic bindings which can be used to + describe Real Time Clock devices in a device tree. + +properties: + $nodename: + pattern: "^rtc(@.*|-[0-9a-f])*$" + + quartz-load-femtofarads: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + The capacitive load of the quartz(x-tal), expressed in femto + Farad (fF). The default value shall be listed (if optional), + and likewise all valid values. + + start-year: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + If provided, the default hardware range supported by the RTC is + shifted so the first usable year is the specified one. + + trickle-diode-disable: + $ref: /schemas/types.yaml#/definitions/flag + description: + Do not use internal trickle charger diode. Should be given if + internal trickle charger diode should be disabled. + + trickle-resistor-ohms: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Selected resistor for trickle charger. Should be given + if trickle charger should be enabled. + + wakeup-source: + $ref: /schemas/types.yaml#/definitions/flag + description: + Enables wake up of host system on alarm. + +... diff --git a/Documentation/devicetree/bindings/rtc/sun6i-rtc.txt b/Documentation/devicetree/bindings/rtc/sun6i-rtc.txt deleted file mode 100644 index 6b732c41392b..000000000000 --- a/Documentation/devicetree/bindings/rtc/sun6i-rtc.txt +++ /dev/null @@ -1,46 +0,0 @@ -* sun6i Real Time Clock - -RTC controller for the Allwinner A31 - -Required properties: -- compatible : Should be one of the following combinations: - - "allwinner,sun6i-a31-rtc" - - "allwinner,sun8i-a23-rtc" - - "allwinner,sun8i-h3-rtc" - - "allwinner,sun8i-r40-rtc", "allwinner,sun8i-h3-rtc" - - "allwinner,sun8i-v3-rtc" - - "allwinner,sun50i-a64-rtc", "allwinner,sun8i-h3-rtc" - - "allwinner,sun50i-h5-rtc" - - Where there are two or more compatible strings, this - denotes the hardware covered by the most specific one - is backward-compatible with the latter ones, and the - implementation for the latter ones can be used, albeit - with reduced functionality. - -- reg : physical base address of the controller and length of - memory mapped region. -- interrupts : IRQ lines for the RTC alarm 0 and alarm 1, in that order. - -Required properties for new device trees -- clocks : phandle to the 32kHz external oscillator -- clock-output-names : names of up to three clock outputs. See below. -- #clock-cells : must be equal to 1. - -The RTC provides the following clocks at the given indices: -- 0: LOSC -- 1: LOSC external output, known as X32KFOUT in the datasheet. - This clock is not available on the A31 and is deprecated for old - device trees still using the "allwinner,sun6i-a31-rtc" compatible. -- 2: InternalOSC, or internal RC oscillator (A64/H3/H5 only) - -Example: - -rtc: rtc@1f00000 { - compatible = "allwinner,sun6i-a31-rtc"; - reg = <0x01f00000 0x400>; - interrupts = <0 40 4>, <0 41 4>; - clock-output-names = "osc32k"; - clocks = <&ext_osc32k>; - #clock-cells = <1>; -}; diff --git a/Documentation/devicetree/bindings/rtc/sunxi-rtc.txt b/Documentation/devicetree/bindings/rtc/sunxi-rtc.txt deleted file mode 100644 index 4a8d79c1cf08..000000000000 --- a/Documentation/devicetree/bindings/rtc/sunxi-rtc.txt +++ /dev/null @@ -1,17 +0,0 @@ -* sun4i/sun7i Real Time Clock - -RTC controller for the Allwinner A10/A20 - -Required properties: -- compatible : Should be "allwinner,sun4i-a10-rtc" or "allwinner,sun7i-a20-rtc" -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: IRQ line for the RTC. - -Example: - -rtc: rtc@1c20d00 { - compatible = "allwinner,sun4i-a10-rtc"; - reg = <0x01c20d00 0x20>; - interrupts = <24>; -}; diff --git a/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml new file mode 100644 index 000000000000..0c12ce9a9b45 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/trivial-rtc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Trivial RTCs + +maintainers: + - Alexandre Belloni <alexandre.belloni@bootlin.com> + +description: | + This is a list of trivial RTC devices that have simple device tree + bindings, consisting only of a compatible field, an address and + possibly an interrupt line. + +allOf: + - $ref: "rtc.yaml#" + +properties: + compatible: + enum: + # AB-RTCMC-32.768kHz-B5ZE-S3: Real Time Clock/Calendar Module with I2C Interface + - abracon,abb5zes3 + # AB-RTCMC-32.768kHz-EOZ9: Real Time Clock/Calendar Module with I2C Interface + - abracon,abeoz9 + # I2C, 32-Bit Binary Counter Watchdog RTC with Trickle Charger and Reset Input/Output + - dallas,ds1374 + # Dallas DS1672 Real-time Clock + - dallas,ds1672 + # Extremely Accurate I²C RTC with Integrated Crystal and SRAM + - dallas,ds3232 + # I2C-BUS INTERFACE REAL TIME CLOCK MODULE + - epson,rx8010 + # I2C-BUS INTERFACE REAL TIME CLOCK MODULE with Battery Backed RAM + - epson,rx8571 + # I2C-BUS INTERFACE REAL TIME CLOCK MODULE + - epson,rx8581 + # Intersil ISL1208 Low Power RTC with Battery Backed SRAM + - isil,isl1208 + # Intersil ISL1218 Low Power RTC with Battery Backed SRAM + - isil,isl1218 + # Intersil ISL12022 Real-time Clock + - isil,isl12022 + # Real Time Clock Module with I2C-Bus + - microcrystal,rv3028 + # Real Time Clock Module with I2C-Bus + - microcrystal,rv3029 + # Real Time Clock + - microcrystal,rv8523 + # Real-time clock + - nxp,pcf2127 + # Real-time clock + - nxp,pcf2129 + # Real-time clock/calendar + - nxp,pcf8563 + # Real-time Clock Module + - pericom,pt7c4338 + # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC + - ricoh,r2025sd + # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC + - ricoh,r2221tl + # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC + - ricoh,rs5c372a + # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC + - ricoh,rs5c372b + # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC + - ricoh,rv5c386 + # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC + - ricoh,rv5c387a + # 2-wire CMOS real-time clock + - sii,s35390a + # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC + - whwave,sd3078 + # Xircom X1205 I2C RTC + - xircom,x1205 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + start-year: true + +required: + - compatible + - reg + +additionalProperties: false + +... diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt index cf759e5f9b10..1214192847ac 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt @@ -21,6 +21,11 @@ edge. Definition: should specify the IRQ used by the remote processor to signal this processor about communication related events +- qcom,remote-pid: + Usage: required for glink-smem + Value type: <u32> + Definition: specifies the identifier of the remote endpoint of this edge + - qcom,rpm-msg-ram: Usage: required for glink-rpm Value type: <prop-encoded-array> diff --git a/Documentation/devicetree/bindings/usb/s3c2410-usb.txt b/Documentation/devicetree/bindings/usb/s3c2410-usb.txt index e45b38ce2986..26c85afd0b53 100644 --- a/Documentation/devicetree/bindings/usb/s3c2410-usb.txt +++ b/Documentation/devicetree/bindings/usb/s3c2410-usb.txt @@ -4,7 +4,7 @@ OHCI Required properties: - compatible: should be "samsung,s3c2410-ohci" for USB host controller - - reg: address and lenght of the controller memory mapped region + - reg: address and length of the controller memory mapped region - interrupts: interrupt number for the USB OHCI controller - clocks: Should reference the bus and host clocks - clock-names: Should contain two strings diff --git a/Documentation/devicetree/bindings/virtio/iommu.txt b/Documentation/devicetree/bindings/virtio/iommu.txt new file mode 100644 index 000000000000..2407fea0651c --- /dev/null +++ b/Documentation/devicetree/bindings/virtio/iommu.txt @@ -0,0 +1,66 @@ +* virtio IOMMU PCI device + +When virtio-iommu uses the PCI transport, its programming interface is +discovered dynamically by the PCI probing infrastructure. However the +device tree statically describes the relation between IOMMU and DMA +masters. Therefore, the PCI root complex that hosts the virtio-iommu +contains a child node representing the IOMMU device explicitly. + +Required properties: + +- compatible: Should be "virtio,pci-iommu" +- reg: PCI address of the IOMMU. As defined in the PCI Bus + Binding reference [1], the reg property is a five-cell + address encoded as (phys.hi phys.mid phys.lo size.hi + size.lo). phys.hi should contain the device's BDF as + 0b00000000 bbbbbbbb dddddfff 00000000. The other cells + should be zero. +- #iommu-cells: Each platform DMA master managed by the IOMMU is assigned + an endpoint ID, described by the "iommus" property [2]. + For virtio-iommu, #iommu-cells must be 1. + +Notes: + +- DMA from the IOMMU device isn't managed by another IOMMU. Therefore the + virtio-iommu node doesn't have an "iommus" property, and is omitted from + the iommu-map property of the root complex. + +Example: + +pcie@10000000 { + compatible = "pci-host-ecam-generic"; + ... + + /* The IOMMU programming interface uses slot 00:01.0 */ + iommu0: iommu@0008 { + compatible = "virtio,pci-iommu"; + reg = <0x00000800 0 0 0 0>; + #iommu-cells = <1>; + }; + + /* + * The IOMMU manages all functions in this PCI domain except + * itself. Omit BDF 00:01.0. + */ + iommu-map = <0x0 &iommu0 0x0 0x8> + <0x9 &iommu0 0x9 0xfff7>; +}; + +pcie@20000000 { + compatible = "pci-host-ecam-generic"; + ... + /* + * The IOMMU also manages all functions from this domain, + * with endpoint IDs 0x10000 - 0x1ffff + */ + iommu-map = <0x0 &iommu0 0x10000 0x10000>; +}; + +ethernet@fe001000 { + ... + /* The IOMMU manages this platform device with endpoint ID 0x20000 */ + iommus = <&iommu0 0x20000>; +}; + +[1] Documentation/devicetree/bindings/pci/pci.txt +[2] Documentation/devicetree/bindings/iommu/iommu.txt diff --git a/Documentation/devicetree/bindings/virtio/mmio.txt b/Documentation/devicetree/bindings/virtio/mmio.txt index 5069c1b8e193..21af30fbb81f 100644 --- a/Documentation/devicetree/bindings/virtio/mmio.txt +++ b/Documentation/devicetree/bindings/virtio/mmio.txt @@ -8,10 +8,40 @@ Required properties: - reg: control registers base address and size including configuration space - interrupts: interrupt generated by the device +Required properties for virtio-iommu: + +- #iommu-cells: When the node corresponds to a virtio-iommu device, it is + linked to DMA masters using the "iommus" or "iommu-map" + properties [1][2]. #iommu-cells specifies the size of the + "iommus" property. For virtio-iommu #iommu-cells must be + 1, each cell describing a single endpoint ID. + +Optional properties: + +- iommus: If the device accesses memory through an IOMMU, it should + have an "iommus" property [1]. Since virtio-iommu itself + does not access memory through an IOMMU, the "virtio,mmio" + node cannot have both an "#iommu-cells" and an "iommus" + property. + Example: virtio_block@3000 { compatible = "virtio,mmio"; reg = <0x3000 0x100>; interrupts = <41>; + + /* Device has endpoint ID 23 */ + iommus = <&viommu 23> } + + viommu: iommu@3100 { + compatible = "virtio,mmio"; + reg = <0x3100 0x100>; + interrupts = <42>; + + #iommu-cells = <1> + } + +[1] Documentation/devicetree/bindings/iommu/iommu.txt +[2] Documentation/devicetree/bindings/pci/pci-iommu.txt diff --git a/Documentation/devicetree/bindings/watchdog/fsl-imx-sc-wdt.txt b/Documentation/devicetree/bindings/watchdog/fsl-imx-sc-wdt.txt deleted file mode 100644 index 02b87e92ae68..000000000000 --- a/Documentation/devicetree/bindings/watchdog/fsl-imx-sc-wdt.txt +++ /dev/null @@ -1,24 +0,0 @@ -* Freescale i.MX System Controller Watchdog - -i.MX system controller watchdog is for i.MX SoCs with system controller inside, -the watchdog is managed by system controller, users can ONLY communicate with -system controller from secure mode for watchdog operations, so Linux i.MX system -controller watchdog driver will call ARM SMC API and trap into ARM-Trusted-Firmware -for watchdog operations, ARM-Trusted-Firmware is running at secure EL3 mode and -it will request system controller to execute the watchdog operation passed from -Linux kernel. - -Required properties: -- compatible: Should be : - "fsl,imx8qxp-sc-wdt" - followed by "fsl,imx-sc-wdt"; - -Optional properties: -- timeout-sec : Contains the watchdog timeout in seconds. - -Examples: - -watchdog { - compatible = "fsl,imx8qxp-sc-wdt", "fsl,imx-sc-wdt"; - timeout-sec = <60>; -}; diff --git a/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt b/Documentation/devicetree/bindings/watchdog/renesas,wdt.txt index 9f365c1a3399..9f365c1a3399 100644 --- a/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt +++ b/Documentation/devicetree/bindings/watchdog/renesas,wdt.txt diff --git a/Documentation/devicetree/bindings/watchdog/sunxi-wdt.txt b/Documentation/devicetree/bindings/watchdog/sunxi-wdt.txt index 46055254e8dd..e65198d82a2b 100644 --- a/Documentation/devicetree/bindings/watchdog/sunxi-wdt.txt +++ b/Documentation/devicetree/bindings/watchdog/sunxi-wdt.txt @@ -6,6 +6,7 @@ Required properties: "allwinner,sun4i-a10-wdt" "allwinner,sun6i-a31-wdt" "allwinner,sun50i-a64-wdt","allwinner,sun6i-a31-wdt" + "allwinner,sun50i-h6-wdt","allwinner,sun6i-a31-wdt" "allwinner,suniv-f1c100s-wdt", "allwinner,sun4i-a10-wdt" - reg : Specifies base physical address and size of the registers. diff --git a/Documentation/driver-api/dmaengine/dmatest.rst b/Documentation/driver-api/dmaengine/dmatest.rst index e78d070bb468..ee268d445d38 100644 --- a/Documentation/driver-api/dmaengine/dmatest.rst +++ b/Documentation/driver-api/dmaengine/dmatest.rst @@ -44,7 +44,8 @@ Example of usage:: dmatest.timeout=2000 dmatest.iterations=1 dmatest.channel=dma0chan0 dmatest.run=1 -Example of multi-channel test usage: +Example of multi-channel test usage (new in the 5.0 kernel):: + % modprobe dmatest % echo 2000 > /sys/module/dmatest/parameters/timeout % echo 1 > /sys/module/dmatest/parameters/iterations @@ -53,15 +54,18 @@ Example of multi-channel test usage: % echo dma0chan2 > /sys/module/dmatest/parameters/channel % echo 1 > /sys/module/dmatest/parameters/run -Note: the channel parameter should always be the last parameter set prior to -running the test (setting run=1), this is because upon setting the channel -parameter, that specific channel is requested using the dmaengine and a thread -is created with the existing parameters. This thread is set as pending -and will be executed once run is set to 1. Any parameters set after the thread -is created are not applied. +.. note:: + For all tests, starting in the 5.0 kernel, either single- or multi-channel, + the channel parameter(s) must be set after all other parameters. It is at + that time that the existing parameter values are acquired for use by the + thread(s). All other parameters are shared. Therefore, if changes are made + to any of the other parameters, and an additional channel specified, the + (shared) parameters used for all threads will use the new values. + After the channels are specified, each thread is set as pending. All threads + begin execution when the run parameter is set to 1. .. hint:: - available channel list could be extracted by running the following command:: + A list of available channels can be found by running the following command:: % ls -1 /sys/class/dma/ @@ -204,6 +208,7 @@ Releasing Channels Channels can be freed by setting run to 0. Example:: + % echo dma0chan1 > /sys/module/dmatest/parameters/channel dmatest: Added 1 threads using dma0chan1 % cat /sys/class/dma/dma0chan1/in_use diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index 4ac99122b5f1..a100bef54952 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -246,6 +246,10 @@ CLOCK devm_clk_get() devm_clk_get_optional() devm_clk_put() + devm_clk_bulk_get() + devm_clk_bulk_get_all() + devm_clk_bulk_get_optional() + devm_get_clk_from_childl() devm_clk_hw_register() devm_of_clk_add_hw_provider() devm_clk_hw_register_clkdev() diff --git a/Documentation/filesystems/coda.txt b/Documentation/filesystems/coda.txt index 61311356025d..545262c167c3 100644 --- a/Documentation/filesystems/coda.txt +++ b/Documentation/filesystems/coda.txt @@ -481,7 +481,10 @@ kernel support. - + struct coda_timespec { + int64_t tv_sec; /* seconds */ + long tv_nsec; /* nanoseconds */ + }; struct coda_vattr { enum coda_vtype va_type; /* vnode type (for create) */ @@ -493,9 +496,9 @@ kernel support. long va_fileid; /* file id */ u_quad_t va_size; /* file size in bytes */ long va_blocksize; /* blocksize preferred for i/o */ - struct timespec va_atime; /* time of last access */ - struct timespec va_mtime; /* time of last modification */ - struct timespec va_ctime; /* time file changed */ + struct coda_timespec va_atime; /* time of last access */ + struct coda_timespec va_mtime; /* time of last modification */ + struct coda_timespec va_ctime; /* time file changed */ u_long va_gen; /* generation number of file */ u_long va_flags; /* flags defined for file */ dev_t va_rdev; /* device special file represents */ diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index 6d2c0d340dea..679729442fd2 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -76,7 +76,7 @@ exposure of uninitialized data through mmap. These filesystems may be used for inspiration: - ext2: see Documentation/filesystems/ext2.txt - ext4: see Documentation/filesystems/ext4/ -- xfs: see Documentation/filesystems/xfs.txt +- xfs: see Documentation/admin-guide/xfs.rst Handling Media Errors diff --git a/Documentation/hwspinlock.txt b/Documentation/hwspinlock.txt index ed640a278185..6f03713b7003 100644 --- a/Documentation/hwspinlock.txt +++ b/Documentation/hwspinlock.txt @@ -136,6 +136,39 @@ The function will never sleep. :: + int hwspin_lock_timeout_raw(struct hwspinlock *hwlock, unsigned int timeout); + +Lock a previously-assigned hwspinlock with a timeout limit (specified in +msecs). If the hwspinlock is already taken, the function will busy loop +waiting for it to be released, but give up when the timeout elapses. + +Caution: User must protect the routine of getting hardware lock with mutex +or spinlock to avoid dead-lock, that will let user can do some time-consuming +or sleepable operations under the hardware lock. + +Returns 0 when successful and an appropriate error code otherwise (most +notably -ETIMEDOUT if the hwspinlock is still busy after timeout msecs). + +The function will never sleep. + +:: + + int hwspin_lock_timeout_in_atomic(struct hwspinlock *hwlock, unsigned int to); + +Lock a previously-assigned hwspinlock with a timeout limit (specified in +msecs). If the hwspinlock is already taken, the function will busy loop +waiting for it to be released, but give up when the timeout elapses. + +This function shall be called only from an atomic context and the timeout +value shall not exceed a few msecs. + +Returns 0 when successful and an appropriate error code otherwise (most +notably -ETIMEDOUT if the hwspinlock is still busy after timeout msecs). + +The function will never sleep. + +:: + int hwspin_trylock(struct hwspinlock *hwlock); @@ -186,6 +219,34 @@ The function will never sleep. :: + int hwspin_trylock_raw(struct hwspinlock *hwlock); + +Attempt to lock a previously-assigned hwspinlock, but immediately fail if +it is already taken. + +Caution: User must protect the routine of getting hardware lock with mutex +or spinlock to avoid dead-lock, that will let user can do some time-consuming +or sleepable operations under the hardware lock. + +Returns 0 on success and an appropriate error code otherwise (most +notably -EBUSY if the hwspinlock was already taken). +The function will never sleep. + +:: + + int hwspin_trylock_in_atomic(struct hwspinlock *hwlock); + +Attempt to lock a previously-assigned hwspinlock, but immediately fail if +it is already taken. + +This function shall be called only from an atomic context. + +Returns 0 on success and an appropriate error code otherwise (most +notably -EBUSY if the hwspinlock was already taken). +The function will never sleep. + +:: + void hwspin_unlock(struct hwspinlock *hwlock); Unlock a previously-locked hwspinlock. Always succeed, and can be called @@ -222,6 +283,26 @@ the given flags. This function will never sleep. :: + void hwspin_unlock_raw(struct hwspinlock *hwlock); + +Unlock a previously-locked hwspinlock. + +The caller should **never** unlock an hwspinlock which is already unlocked. +Doing so is considered a bug (there is no protection against this). +This function will never sleep. + +:: + + void hwspin_unlock_in_atomic(struct hwspinlock *hwlock); + +Unlock a previously-locked hwspinlock. + +The caller should **never** unlock an hwspinlock which is already unlocked. +Doing so is considered a bug (there is no protection against this). +This function will never sleep. + +:: + int hwspin_lock_get_id(struct hwspinlock *hwlock); Retrieve id number of a given hwspinlock. This is needed when an diff --git a/Documentation/power/pm_qos_interface.rst b/Documentation/power/pm_qos_interface.rst index 945fc6d760c9..69921f072ce1 100644 --- a/Documentation/power/pm_qos_interface.rst +++ b/Documentation/power/pm_qos_interface.rst @@ -129,7 +129,7 @@ int dev_pm_qos_remove_request(handle): and call the notification trees if the target was changed as a result of removing the request. -s32 dev_pm_qos_read_value(device): +s32 dev_pm_qos_read_value(device, type): Returns the aggregated value for a given device's constraints list. enum pm_qos_flags_status dev_pm_qos_flags(device, mask) @@ -176,12 +176,14 @@ Notification mechanisms: The per-device PM QoS framework has a per-device notification tree. -int dev_pm_qos_add_notifier(device, notifier): - Adds a notification callback function for the device. +int dev_pm_qos_add_notifier(device, notifier, type): + Adds a notification callback function for the device for a particular request + type. + The callback is called when the aggregated value of the device constraints list - is changed (for resume latency device PM QoS only). + is changed. -int dev_pm_qos_remove_notifier(device, notifier): +int dev_pm_qos_remove_notifier(device, notifier, type): Removes the notification callback function for the device. diff --git a/Documentation/remoteproc.txt b/Documentation/remoteproc.txt index 77fb03acdbb4..03c3d2e568b0 100644 --- a/Documentation/remoteproc.txt +++ b/Documentation/remoteproc.txt @@ -314,6 +314,8 @@ Here are the various resource types that are currently supported:: * @RSC_VDEV: declare support for a virtio device, and serve as its * virtio header. * @RSC_LAST: just keep this one at the end + * @RSC_VENDOR_START: start of the vendor specific resource types range + * @RSC_VENDOR_END: end of the vendor specific resource types range * * Please note that these values are used as indices to the rproc_handle_rsc * lookup table, so please keep them sane. Moreover, @RSC_LAST is used to @@ -321,11 +323,13 @@ Here are the various resource types that are currently supported:: * please update it as needed. */ enum fw_resource_type { - RSC_CARVEOUT = 0, - RSC_DEVMEM = 1, - RSC_TRACE = 2, - RSC_VDEV = 3, - RSC_LAST = 4, + RSC_CARVEOUT = 0, + RSC_DEVMEM = 1, + RSC_TRACE = 2, + RSC_VDEV = 3, + RSC_LAST = 4, + RSC_VENDOR_START = 128, + RSC_VENDOR_END = 512, }; For more details regarding a specific resource type, please see its diff --git a/Documentation/thermal/cpu-cooling-api.txt b/Documentation/thermal/cpu-cooling-api.rst index 7df567eaea1a..645d914c45a6 100644 --- a/Documentation/thermal/cpu-cooling-api.txt +++ b/Documentation/thermal/cpu-cooling-api.rst @@ -1,5 +1,6 @@ +======================= CPU cooling APIs How To -=================================== +======================= Written by Amit Daniel Kachhap <amit.kachhap@linaro.org> @@ -8,40 +9,54 @@ Updated: 6 Jan 2015 Copyright (c) 2012 Samsung Electronics Co., Ltd(http://www.samsung.com) 0. Introduction +=============== The generic cpu cooling(freq clipping) provides registration/unregistration APIs to the caller. The binding of the cooling devices to the trip point is left for the user. The registration APIs returns the cooling device pointer. 1. cpu cooling APIs +=================== 1.1 cpufreq registration/unregistration APIs -1.1.1 struct thermal_cooling_device *cpufreq_cooling_register( - struct cpumask *clip_cpus) +-------------------------------------------- + + :: + + struct thermal_cooling_device + *cpufreq_cooling_register(struct cpumask *clip_cpus) This interface function registers the cpufreq cooling device with the name "thermal-cpufreq-%x". This api can support multiple instances of cpufreq cooling devices. - clip_cpus: cpumask of cpus where the frequency constraints will happen. + clip_cpus: + cpumask of cpus where the frequency constraints will happen. + + :: -1.1.2 struct thermal_cooling_device *of_cpufreq_cooling_register( - struct cpufreq_policy *policy) + struct thermal_cooling_device + *of_cpufreq_cooling_register(struct cpufreq_policy *policy) This interface function registers the cpufreq cooling device with the name "thermal-cpufreq-%x" linking it with a device tree node, in order to bind it via the thermal DT code. This api can support multiple instances of cpufreq cooling devices. - policy: CPUFreq policy. + policy: + CPUFreq policy. + + + :: -1.1.3 void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) + void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) This interface function unregisters the "thermal-cpufreq-%x" cooling device. cdev: Cooling device pointer which has to be unregistered. 2. Power models +=============== The power API registration functions provide a simple power model for CPUs. The current power is calculated as dynamic power (static power isn't @@ -65,9 +80,9 @@ For a given processor implementation the primary factors are: variation. In pathological cases this variation can be significant, but typically it is of a much lesser impact than the factors above. -A high level dynamic power consumption model may then be represented as: +A high level dynamic power consumption model may then be represented as:: -Pdyn = f(run) * Voltage^2 * Frequency * Utilisation + Pdyn = f(run) * Voltage^2 * Frequency * Utilisation f(run) here represents the described execution behaviour and its result has a units of Watts/Hz/Volt^2 (this often expressed in @@ -80,9 +95,9 @@ factors. Therefore, in initial implementation that contribution is represented as a constant coefficient. This is a simplification consistent with the relative contribution to overall power variation. -In this simplified representation our model becomes: +In this simplified representation our model becomes:: -Pdyn = Capacitance * Voltage^2 * Frequency * Utilisation + Pdyn = Capacitance * Voltage^2 * Frequency * Utilisation Where `capacitance` is a constant that represents an indicative running time dynamic power coefficient in fundamental units of diff --git a/Documentation/thermal/exynos_thermal b/Documentation/thermal/exynos_thermal.rst index 9010c4416967..5bd556566c70 100644 --- a/Documentation/thermal/exynos_thermal +++ b/Documentation/thermal/exynos_thermal.rst @@ -1,8 +1,11 @@ +======================== Kernel driver exynos_tmu -================= +======================== Supported chips: + * ARM SAMSUNG EXYNOS4, EXYNOS5 series of SoC + Datasheet: Not publicly available Authors: Donggeun Kim <dg77.kim@samsung.com> @@ -19,32 +22,39 @@ Temperature can be taken from the temperature code. There are three equations converting from temperature to temperature code. The three equations are: - 1. Two point trimming + 1. Two point trimming:: + Tc = (T - 25) * (TI2 - TI1) / (85 - 25) + TI1 - 2. One point trimming + 2. One point trimming:: + Tc = T + TI1 - 25 - 3. No trimming + 3. No trimming:: + Tc = T + 50 - Tc: Temperature code, T: Temperature, - TI1: Trimming info for 25 degree Celsius (stored at TRIMINFO register) + Tc: + Temperature code, T: Temperature, + TI1: + Trimming info for 25 degree Celsius (stored at TRIMINFO register) Temperature code measured at 25 degree Celsius which is unchanged - TI2: Trimming info for 85 degree Celsius (stored at TRIMINFO register) + TI2: + Trimming info for 85 degree Celsius (stored at TRIMINFO register) Temperature code measured at 85 degree Celsius which is unchanged TMU(Thermal Management Unit) in EXYNOS4/5 generates interrupt when temperature exceeds pre-defined levels. The maximum number of configurable threshold is five. -The threshold levels are defined as follows: +The threshold levels are defined as follows:: + Level_0: current temperature > trigger_level_0 + threshold Level_1: current temperature > trigger_level_1 + threshold Level_2: current temperature > trigger_level_2 + threshold Level_3: current temperature > trigger_level_3 + threshold - The threshold and each trigger_level are set - through the corresponding registers. +The threshold and each trigger_level are set +through the corresponding registers. When an interrupt occurs, this driver notify kernel thermal framework with the function exynos_report_trigger. @@ -54,24 +64,27 @@ it can be used to synchronize the cooling action. TMU driver description: ----------------------- -The exynos thermal driver is structured as, +The exynos thermal driver is structured as:: Kernel Core thermal framework (thermal_core.c, step_wise.c, cpu_cooling.c) ^ | | -TMU configuration data -------> TMU Driver <------> Exynos Core thermal wrapper -(exynos_tmu_data.c) (exynos_tmu.c) (exynos_thermal_common.c) -(exynos_tmu_data.h) (exynos_tmu.h) (exynos_thermal_common.h) + TMU configuration data -----> TMU Driver <----> Exynos Core thermal wrapper + (exynos_tmu_data.c) (exynos_tmu.c) (exynos_thermal_common.c) + (exynos_tmu_data.h) (exynos_tmu.h) (exynos_thermal_common.h) -a) TMU configuration data: This consist of TMU register offsets/bitfields +a) TMU configuration data: + This consist of TMU register offsets/bitfields described through structure exynos_tmu_registers. Also several other platform data (struct exynos_tmu_platform_data) members are used to configure the TMU. -b) TMU driver: This component initialises the TMU controller and sets different +b) TMU driver: + This component initialises the TMU controller and sets different thresholds. It invokes core thermal implementation with the call exynos_report_trigger. -c) Exynos Core thermal wrapper: This provides 3 wrapper function to use the +c) Exynos Core thermal wrapper: + This provides 3 wrapper function to use the Kernel core thermal framework. They are exynos_unregister_thermal, exynos_register_thermal and exynos_report_trigger. diff --git a/Documentation/thermal/exynos_thermal_emulation b/Documentation/thermal/exynos_thermal_emulation deleted file mode 100644 index b15efec6ca28..000000000000 --- a/Documentation/thermal/exynos_thermal_emulation +++ /dev/null @@ -1,53 +0,0 @@ -EXYNOS EMULATION MODE -======================== - -Copyright (C) 2012 Samsung Electronics - -Written by Jonghwa Lee <jonghwa3.lee@samsung.com> - -Description ------------ - -Exynos 4x12 (4212, 4412) and 5 series provide emulation mode for thermal management unit. -Thermal emulation mode supports software debug for TMU's operation. User can set temperature -manually with software code and TMU will read current temperature from user value not from -sensor's value. - -Enabling CONFIG_THERMAL_EMULATION option will make this support available. -When it's enabled, sysfs node will be created as -/sys/devices/virtual/thermal/thermal_zone'zone id'/emul_temp. - -The sysfs node, 'emul_node', will contain value 0 for the initial state. When you input any -temperature you want to update to sysfs node, it automatically enable emulation mode and -current temperature will be changed into it. -(Exynos also supports user changeable delay time which would be used to delay of - changing temperature. However, this node only uses same delay of real sensing time, 938us.) - -Exynos emulation mode requires synchronous of value changing and enabling. It means when you -want to update the any value of delay or next temperature, then you have to enable emulation -mode at the same time. (Or you have to keep the mode enabling.) If you don't, it fails to -change the value to updated one and just use last succeessful value repeatedly. That's why -this node gives users the right to change termerpature only. Just one interface makes it more -simply to use. - -Disabling emulation mode only requires writing value 0 to sysfs node. - - -TEMP 120 | - | - 100 | - | - 80 | - | +----------- - 60 | | | - | +-------------| | - 40 | | | | - | | | | - 20 | | | +---------- - | | | | | - 0 |______________|_____________|__________|__________|_________ - A A A A TIME - |<----->| |<----->| |<----->| | - | 938us | | | | | | -emulation : 0 50 | 70 | 20 | 0 -current temp : sensor 50 70 20 sensor diff --git a/Documentation/thermal/exynos_thermal_emulation.rst b/Documentation/thermal/exynos_thermal_emulation.rst new file mode 100644 index 000000000000..c21d10838bc5 --- /dev/null +++ b/Documentation/thermal/exynos_thermal_emulation.rst @@ -0,0 +1,61 @@ +===================== +Exynos Emulation Mode +===================== + +Copyright (C) 2012 Samsung Electronics + +Written by Jonghwa Lee <jonghwa3.lee@samsung.com> + +Description +----------- + +Exynos 4x12 (4212, 4412) and 5 series provide emulation mode for thermal +management unit. Thermal emulation mode supports software debug for +TMU's operation. User can set temperature manually with software code +and TMU will read current temperature from user value not from sensor's +value. + +Enabling CONFIG_THERMAL_EMULATION option will make this support +available. When it's enabled, sysfs node will be created as +/sys/devices/virtual/thermal/thermal_zone'zone id'/emul_temp. + +The sysfs node, 'emul_node', will contain value 0 for the initial state. +When you input any temperature you want to update to sysfs node, it +automatically enable emulation mode and current temperature will be +changed into it. + +(Exynos also supports user changeable delay time which would be used to +delay of changing temperature. However, this node only uses same delay +of real sensing time, 938us.) + +Exynos emulation mode requires synchronous of value changing and +enabling. It means when you want to update the any value of delay or +next temperature, then you have to enable emulation mode at the same +time. (Or you have to keep the mode enabling.) If you don't, it fails to +change the value to updated one and just use last succeessful value +repeatedly. That's why this node gives users the right to change +termerpature only. Just one interface makes it more simply to use. + +Disabling emulation mode only requires writing value 0 to sysfs node. + +:: + + + TEMP 120 | + | + 100 | + | + 80 | + | +----------- + 60 | | | + | +-------------| | + 40 | | | | + | | | | + 20 | | | +---------- + | | | | | + 0 |______________|_____________|__________|__________|_________ + A A A A TIME + |<----->| |<----->| |<----->| | + | 938us | | | | | | + emulation : 0 50 | 70 | 20 | 0 + current temp: sensor 50 70 20 sensor diff --git a/Documentation/thermal/index.rst b/Documentation/thermal/index.rst new file mode 100644 index 000000000000..8c1c00146cad --- /dev/null +++ b/Documentation/thermal/index.rst @@ -0,0 +1,18 @@ +:orphan: + +======= +Thermal +======= + +.. toctree:: + :maxdepth: 1 + + cpu-cooling-api + sysfs-api + power_allocator + + exynos_thermal + exynos_thermal_emulation + intel_powerclamp + nouveau_thermal + x86_pkg_temperature_thermal diff --git a/Documentation/thermal/intel_powerclamp.txt b/Documentation/thermal/intel_powerclamp.rst index b5df21168fbc..3f6dfb0b3ea6 100644 --- a/Documentation/thermal/intel_powerclamp.txt +++ b/Documentation/thermal/intel_powerclamp.rst @@ -1,10 +1,13 @@ - ======================= - INTEL POWERCLAMP DRIVER - ======================= -By: Arjan van de Ven <arjan@linux.intel.com> - Jacob Pan <jacob.jun.pan@linux.intel.com> +======================= +Intel Powerclamp Driver +======================= + +By: + - Arjan van de Ven <arjan@linux.intel.com> + - Jacob Pan <jacob.jun.pan@linux.intel.com> + +.. Contents: -Contents: (*) Introduction - Goals and Objectives @@ -23,7 +26,6 @@ Contents: - Generic Thermal Layer (sysfs) - Kernel APIs (TBD) -============ INTRODUCTION ============ @@ -47,7 +49,6 @@ scalability, and user experience. In many cases, clear advantage is shown over taking the CPU offline or modulating the CPU clock. -=================== THEORY OF OPERATION =================== @@ -57,11 +58,12 @@ Idle Injection On modern Intel processors (Nehalem or later), package level C-state residency is available in MSRs, thus also available to the kernel. -These MSRs are: - #define MSR_PKG_C2_RESIDENCY 0x60D - #define MSR_PKG_C3_RESIDENCY 0x3F8 - #define MSR_PKG_C6_RESIDENCY 0x3F9 - #define MSR_PKG_C7_RESIDENCY 0x3FA +These MSRs are:: + + #define MSR_PKG_C2_RESIDENCY 0x60D + #define MSR_PKG_C3_RESIDENCY 0x3F8 + #define MSR_PKG_C6_RESIDENCY 0x3F9 + #define MSR_PKG_C7_RESIDENCY 0x3FA If the kernel can also inject idle time to the system, then a closed-loop control system can be established that manages package @@ -96,19 +98,21 @@ are not masked. Tests show that the extra wakeups from scheduler tick have a dramatic impact on the effectiveness of the powerclamp driver on large scale systems (Westmere system with 80 processors). -CPU0 - ____________ ____________ -kidle_inject/0 | sleep | mwait | sleep | - _________| |________| |_______ - duration -CPU1 - ____________ ____________ -kidle_inject/1 | sleep | mwait | sleep | - _________| |________| |_______ - ^ - | - | - roundup(jiffies, interval) +:: + + CPU0 + ____________ ____________ + kidle_inject/0 | sleep | mwait | sleep | + _________| |________| |_______ + duration + CPU1 + ____________ ____________ + kidle_inject/1 | sleep | mwait | sleep | + _________| |________| |_______ + ^ + | + | + roundup(jiffies, interval) Only one CPU is allowed to collect statistics and update global control parameters. This CPU is referred to as the controlling CPU in @@ -148,7 +152,7 @@ b) determine the amount of compensation needed at each target ratio Compensation to each target ratio consists of two parts: - a) steady state error compensation + a) steady state error compensation This is to offset the error occurring when the system can enter idle without extra wakeups (such as external interrupts). @@ -158,41 +162,42 @@ Compensation to each target ratio consists of two parts: slowing down CPU activities. A debugfs file is provided for the user to examine compensation -progress and results, such as on a Westmere system. -[jacob@nex01 ~]$ cat -/sys/kernel/debug/intel_powerclamp/powerclamp_calib -controlling cpu: 0 -pct confidence steady dynamic (compensation) -0 0 0 0 -1 1 0 0 -2 1 1 0 -3 3 1 0 -4 3 1 0 -5 3 1 0 -6 3 1 0 -7 3 1 0 -8 3 1 0 -... -30 3 2 0 -31 3 2 0 -32 3 1 0 -33 3 2 0 -34 3 1 0 -35 3 2 0 -36 3 1 0 -37 3 2 0 -38 3 1 0 -39 3 2 0 -40 3 3 0 -41 3 1 0 -42 3 2 0 -43 3 1 0 -44 3 1 0 -45 3 2 0 -46 3 3 0 -47 3 0 0 -48 3 2 0 -49 3 3 0 +progress and results, such as on a Westmere system:: + + [jacob@nex01 ~]$ cat + /sys/kernel/debug/intel_powerclamp/powerclamp_calib + controlling cpu: 0 + pct confidence steady dynamic (compensation) + 0 0 0 0 + 1 1 0 0 + 2 1 1 0 + 3 3 1 0 + 4 3 1 0 + 5 3 1 0 + 6 3 1 0 + 7 3 1 0 + 8 3 1 0 + ... + 30 3 2 0 + 31 3 2 0 + 32 3 1 0 + 33 3 2 0 + 34 3 1 0 + 35 3 2 0 + 36 3 1 0 + 37 3 2 0 + 38 3 1 0 + 39 3 2 0 + 40 3 3 0 + 41 3 1 0 + 42 3 2 0 + 43 3 1 0 + 44 3 1 0 + 45 3 2 0 + 46 3 3 0 + 47 3 0 0 + 48 3 2 0 + 49 3 3 0 Calibration occurs during runtime. No offline method is available. Steady state compensation is used only when confidence levels of all @@ -217,9 +222,8 @@ keeps track of clamping kernel threads, even after they are migrated to other CPUs, after a CPU offline event. -===================== Performance Analysis -===================== +==================== This section describes the general performance data collected on multiple systems, including Westmere (80P) and Ivy Bridge (4P, 8P). @@ -257,16 +261,15 @@ achieve up to 40% better performance per watt. (measured by a spin counter summed over per CPU counting threads spawned for all running CPUs). -==================== Usage and Interfaces ==================== The powerclamp driver is registered to the generic thermal layer as a -cooling device. Currently, it’s not bound to any thermal zones. +cooling device. Currently, it’s not bound to any thermal zones:: -jacob@chromoly:/sys/class/thermal/cooling_device14$ grep . * -cur_state:0 -max_state:50 -type:intel_powerclamp + jacob@chromoly:/sys/class/thermal/cooling_device14$ grep . * + cur_state:0 + max_state:50 + type:intel_powerclamp cur_state allows user to set the desired idle percentage. Writing 0 to cur_state will stop idle injection. Writing a value between 1 and @@ -278,9 +281,9 @@ cur_state returns value -1 instead of 0 which is to avoid confusing 100% busy state with the disabled state. Example usage: -- To inject 25% idle time -$ sudo sh -c "echo 25 > /sys/class/thermal/cooling_device80/cur_state -" +- To inject 25% idle time:: + + $ sudo sh -c "echo 25 > /sys/class/thermal/cooling_device80/cur_state If the system is not busy and has more than 25% idle time already, then the powerclamp driver will not start idle injection. Using Top @@ -292,23 +295,23 @@ idle time is accounted as normal idle in that common code path is taken as the idle task. In this example, 24.1% idle is shown. This helps the system admin or -user determine the cause of slowdown, when a powerclamp driver is in action. - - -Tasks: 197 total, 1 running, 196 sleeping, 0 stopped, 0 zombie -Cpu(s): 71.2%us, 4.7%sy, 0.0%ni, 24.1%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st -Mem: 3943228k total, 1689632k used, 2253596k free, 74960k buffers -Swap: 4087804k total, 0k used, 4087804k free, 945336k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND - 3352 jacob 20 0 262m 644 428 S 286 0.0 0:17.16 spin - 3341 root -51 0 0 0 0 D 25 0.0 0:01.62 kidle_inject/0 - 3344 root -51 0 0 0 0 D 25 0.0 0:01.60 kidle_inject/3 - 3342 root -51 0 0 0 0 D 25 0.0 0:01.61 kidle_inject/1 - 3343 root -51 0 0 0 0 D 25 0.0 0:01.60 kidle_inject/2 - 2935 jacob 20 0 696m 125m 35m S 5 3.3 0:31.11 firefox - 1546 root 20 0 158m 20m 6640 S 3 0.5 0:26.97 Xorg - 2100 jacob 20 0 1223m 88m 30m S 3 2.3 0:23.68 compiz +user determine the cause of slowdown, when a powerclamp driver is in action:: + + + Tasks: 197 total, 1 running, 196 sleeping, 0 stopped, 0 zombie + Cpu(s): 71.2%us, 4.7%sy, 0.0%ni, 24.1%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st + Mem: 3943228k total, 1689632k used, 2253596k free, 74960k buffers + Swap: 4087804k total, 0k used, 4087804k free, 945336k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 3352 jacob 20 0 262m 644 428 S 286 0.0 0:17.16 spin + 3341 root -51 0 0 0 0 D 25 0.0 0:01.62 kidle_inject/0 + 3344 root -51 0 0 0 0 D 25 0.0 0:01.60 kidle_inject/3 + 3342 root -51 0 0 0 0 D 25 0.0 0:01.61 kidle_inject/1 + 3343 root -51 0 0 0 0 D 25 0.0 0:01.60 kidle_inject/2 + 2935 jacob 20 0 696m 125m 35m S 5 3.3 0:31.11 firefox + 1546 root 20 0 158m 20m 6640 S 3 0.5 0:26.97 Xorg + 2100 jacob 20 0 1223m 88m 30m S 3 2.3 0:23.68 compiz Tests have shown that by using the powerclamp driver as a cooling device, a PID based userspace thermal controller can manage to diff --git a/Documentation/thermal/nouveau_thermal b/Documentation/thermal/nouveau_thermal.rst index 6e17a11efcb0..37255fd6735d 100644 --- a/Documentation/thermal/nouveau_thermal +++ b/Documentation/thermal/nouveau_thermal.rst @@ -1,13 +1,15 @@ +===================== Kernel driver nouveau -=================== +===================== Supported chips: + * NV43+ Authors: Martin Peres (mupuf) <martin.peres@free.fr> Description ---------- +----------- This driver allows to read the GPU core temperature, drive the GPU fan and set temperature alarms. @@ -19,20 +21,25 @@ interface is likely not to work. This document may then not cover your situation entirely. Temperature management --------------------- +---------------------- Temperature is exposed under as a read-only HWMON attribute temp1_input. In order to protect the GPU from overheating, Nouveau supports 4 configurable temperature thresholds: - * Fan_boost: Fan speed is set to 100% when reaching this temperature; - * Downclock: The GPU will be downclocked to reduce its power dissipation; - * Critical: The GPU is put on hold to further lower power dissipation; - * Shutdown: Shut the computer down to protect your GPU. + * Fan_boost: + Fan speed is set to 100% when reaching this temperature; + * Downclock: + The GPU will be downclocked to reduce its power dissipation; + * Critical: + The GPU is put on hold to further lower power dissipation; + * Shutdown: + Shut the computer down to protect your GPU. -WARNING: Some of these thresholds may not be used by Nouveau depending -on your chipset. +WARNING: + Some of these thresholds may not be used by Nouveau depending + on your chipset. The default value for these thresholds comes from the GPU's vbios. These thresholds can be configured thanks to the following HWMON attributes: @@ -46,19 +53,24 @@ NOTE: Remember that the values are stored as milli degrees Celsius. Don't forget to multiply! Fan management ------------- +-------------- Not all cards have a drivable fan. If you do, then the following HWMON attributes should be available: - * pwm1_enable: Current fan management mode (NONE, MANUAL or AUTO); - * pwm1: Current PWM value (power percentage); - * pwm1_min: The minimum PWM speed allowed; - * pwm1_max: The maximum PWM speed allowed (bypassed when hitting Fan_boost); + * pwm1_enable: + Current fan management mode (NONE, MANUAL or AUTO); + * pwm1: + Current PWM value (power percentage); + * pwm1_min: + The minimum PWM speed allowed; + * pwm1_max: + The maximum PWM speed allowed (bypassed when hitting Fan_boost); You may also have the following attribute: - * fan1_input: Speed in RPM of your fan. + * fan1_input: + Speed in RPM of your fan. Your fan can be driven in different modes: @@ -66,14 +78,16 @@ Your fan can be driven in different modes: * 1: The fan can be driven in manual (use pwm1 to change the speed); * 2; The fan is driven automatically depending on the temperature. -NOTE: Be sure to use the manual mode if you want to drive the fan speed manually +NOTE: + Be sure to use the manual mode if you want to drive the fan speed manually -NOTE2: When operating in manual mode outside the vbios-defined -[PWM_min, PWM_max] range, the reported fan speed (RPM) may not be accurate -depending on your hardware. +NOTE2: + When operating in manual mode outside the vbios-defined + [PWM_min, PWM_max] range, the reported fan speed (RPM) may not be accurate + depending on your hardware. Bug reports ---------- +----------- Thermal management on Nouveau is new and may not work on all cards. If you have inquiries, please ping mupuf on IRC (#nouveau, freenode). diff --git a/Documentation/thermal/power_allocator.txt b/Documentation/thermal/power_allocator.rst index 9fb0ff06dca9..67b6a3297238 100644 --- a/Documentation/thermal/power_allocator.txt +++ b/Documentation/thermal/power_allocator.rst @@ -1,3 +1,4 @@ +================================= Power allocator governor tunables ================================= @@ -25,36 +26,36 @@ temperature as the control input and power as the controlled output: P_max = k_p * e + k_i * err_integral + k_d * diff_err + sustainable_power where - e = desired_temperature - current_temperature - err_integral is the sum of previous errors - diff_err = e - previous_error - -It is similar to the one depicted below: - - k_d - | -current_temp | - | v - | +----------+ +---+ - | +----->| diff_err |-->| X |------+ - | | +----------+ +---+ | - | | | tdp actor - | | k_i | | get_requested_power() - | | | | | | | - | | | | | | | ... - v | v v v v v - +---+ | +-------+ +---+ +---+ +---+ +----------+ - | S |-------+----->| sum e |----->| X |--->| S |-->| S |-->|power | - +---+ | +-------+ +---+ +---+ +---+ |allocation| - ^ | ^ +----------+ - | | | | | - | | +---+ | | | - | +------->| X |-------------------+ v v - | +---+ granted performance -desired_temperature ^ - | - | - k_po/k_pu + - e = desired_temperature - current_temperature + - err_integral is the sum of previous errors + - diff_err = e - previous_error + +It is similar to the one depicted below:: + + k_d + | + current_temp | + | v + | +----------+ +---+ + | +----->| diff_err |-->| X |------+ + | | +----------+ +---+ | + | | | tdp actor + | | k_i | | get_requested_power() + | | | | | | | + | | | | | | | ... + v | v v v v v + +---+ | +-------+ +---+ +---+ +---+ +----------+ + | S |-----+----->| sum e |----->| X |--->| S |-->| S |-->|power | + +---+ | +-------+ +---+ +---+ +---+ |allocation| + ^ | ^ +----------+ + | | | | | + | | +---+ | | | + | +------->| X |-------------------+ v v + | +---+ granted performance + desired_temperature ^ + | + | + k_po/k_pu Sustainable power ----------------- @@ -73,7 +74,7 @@ is typically 2000mW, while on a 10" tablet is around 4500mW (may vary depending on screen size). If you are using device tree, do add it as a property of the -thermal-zone. For example: +thermal-zone. For example:: thermal-zones { soc_thermal { @@ -85,7 +86,7 @@ thermal-zone. For example: Instead, if the thermal zone is registered from the platform code, pass a `thermal_zone_params` that has a `sustainable_power`. If no `thermal_zone_params` were being passed, then something like below -will suffice: +will suffice:: static const struct thermal_zone_params tz_params = { .sustainable_power = 3500, @@ -112,18 +113,18 @@ available capacity at a low temperature. On the other hand, a high value of `k_pu` will result in the governor granting very high power while temperature is low, and may lead to temperature overshooting. -The default value for `k_pu` is: +The default value for `k_pu` is:: 2 * sustainable_power / (desired_temperature - switch_on_temp) This means that at `switch_on_temp` the output of the controller's proportional term will be 2 * `sustainable_power`. The default value -for `k_po` is: +for `k_po` is:: sustainable_power / (desired_temperature - switch_on_temp) Focusing on the proportional and feed forward values of the PID -controller equation we have: +controller equation we have:: P_max = k_p * e + sustainable_power @@ -134,21 +135,23 @@ is the desired one, then the proportional component is zero and thermal equilibrium under constant load. `sustainable_power` is only an estimate, which is the reason for closed-loop control such as this. -Expanding `k_pu` we get: +Expanding `k_pu` we get:: + P_max = 2 * sustainable_power * (T_set - T) / (T_set - T_on) + - sustainable_power + sustainable_power -where - T_set is the desired temperature - T is the current temperature - T_on is the switch on temperature +where: + + - T_set is the desired temperature + - T is the current temperature + - T_on is the switch on temperature When the current temperature is the switch_on temperature, the above -formula becomes: +formula becomes:: P_max = 2 * sustainable_power * (T_set - T_on) / (T_set - T_on) + - sustainable_power = 2 * sustainable_power + sustainable_power = - 3 * sustainable_power + sustainable_power = 2 * sustainable_power + sustainable_power = + 3 * sustainable_power Therefore, the proportional term alone linearly decreases power from 3 * `sustainable_power` to `sustainable_power` as the temperature @@ -178,11 +181,18 @@ Cooling device power API Cooling devices controlled by this governor must supply the additional "power" API in their `cooling_device_ops`. It consists on three ops: -1. int get_requested_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *power); -@cdev: The `struct thermal_cooling_device` pointer -@tz: thermal zone in which we are currently operating -@power: pointer in which to store the calculated power +1. :: + + int get_requested_power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, u32 *power); + + +@cdev: + The `struct thermal_cooling_device` pointer +@tz: + thermal zone in which we are currently operating +@power: + pointer in which to store the calculated power `get_requested_power()` calculates the power requested by the device in milliwatts and stores it in @power . It should return 0 on @@ -190,23 +200,37 @@ success, -E* on failure. This is currently used by the power allocator governor to calculate how much power to give to each cooling device. -2. int state2power(struct thermal_cooling_device *cdev, struct - thermal_zone_device *tz, unsigned long state, u32 *power); -@cdev: The `struct thermal_cooling_device` pointer -@tz: thermal zone in which we are currently operating -@state: A cooling device state -@power: pointer in which to store the equivalent power +2. :: + + int state2power(struct thermal_cooling_device *cdev, struct + thermal_zone_device *tz, unsigned long state, + u32 *power); + +@cdev: + The `struct thermal_cooling_device` pointer +@tz: + thermal zone in which we are currently operating +@state: + A cooling device state +@power: + pointer in which to store the equivalent power Convert cooling device state @state into power consumption in milliwatts and store it in @power. It should return 0 on success, -E* on failure. This is currently used by thermal core to calculate the maximum power that an actor can consume. -3. int power2state(struct thermal_cooling_device *cdev, u32 power, - unsigned long *state); -@cdev: The `struct thermal_cooling_device` pointer -@power: power in milliwatts -@state: pointer in which to store the resulting state +3. :: + + int power2state(struct thermal_cooling_device *cdev, u32 power, + unsigned long *state); + +@cdev: + The `struct thermal_cooling_device` pointer +@power: + power in milliwatts +@state: + pointer in which to store the resulting state Calculate a cooling device state that would make the device consume at most @power mW and store it in @state. It should return 0 on success, diff --git a/Documentation/thermal/sysfs-api.txt b/Documentation/thermal/sysfs-api.rst index c3fa500df92c..e4930761d3e5 100644 --- a/Documentation/thermal/sysfs-api.txt +++ b/Documentation/thermal/sysfs-api.rst @@ -1,3 +1,4 @@ +=================================== Generic Thermal Sysfs driver How To =================================== @@ -9,6 +10,7 @@ Copyright (c) 2008 Intel Corporation 0. Introduction +=============== The generic thermal sysfs provides a set of interfaces for thermal zone devices (sensors) and thermal cooling devices (fan, processor...) to register @@ -25,59 +27,90 @@ An intelligent thermal management application can make decisions based on inputs from thermal zone attributes (the current temperature and trip point temperature) and throttle appropriate devices. -[0-*] denotes any positive number starting from 0 -[1-*] denotes any positive number starting from 1 +- `[0-*]` denotes any positive number starting from 0 +- `[1-*]` denotes any positive number starting from 1 1. thermal sysfs driver interface functions +=========================================== 1.1 thermal zone device interface -1.1.1 struct thermal_zone_device *thermal_zone_device_register(char *type, - int trips, int mask, void *devdata, - struct thermal_zone_device_ops *ops, - const struct thermal_zone_params *tzp, - int passive_delay, int polling_delay)) +--------------------------------- + + :: + + struct thermal_zone_device + *thermal_zone_device_register(char *type, + int trips, int mask, void *devdata, + struct thermal_zone_device_ops *ops, + const struct thermal_zone_params *tzp, + int passive_delay, int polling_delay)) This interface function adds a new thermal zone device (sensor) to - /sys/class/thermal folder as thermal_zone[0-*]. It tries to bind all the + /sys/class/thermal folder as `thermal_zone[0-*]`. It tries to bind all the thermal cooling devices registered at the same time. - type: the thermal zone type. - trips: the total number of trip points this thermal zone supports. - mask: Bit string: If 'n'th bit is set, then trip point 'n' is writeable. - devdata: device private data - ops: thermal zone device call-backs. - .bind: bind the thermal zone device with a thermal cooling device. - .unbind: unbind the thermal zone device with a thermal cooling device. - .get_temp: get the current temperature of the thermal zone. - .set_trips: set the trip points window. Whenever the current temperature + type: + the thermal zone type. + trips: + the total number of trip points this thermal zone supports. + mask: + Bit string: If 'n'th bit is set, then trip point 'n' is writeable. + devdata: + device private data + ops: + thermal zone device call-backs. + + .bind: + bind the thermal zone device with a thermal cooling device. + .unbind: + unbind the thermal zone device with a thermal cooling device. + .get_temp: + get the current temperature of the thermal zone. + .set_trips: + set the trip points window. Whenever the current temperature is updated, the trip points immediately below and above the current temperature are found. - .get_mode: get the current mode (enabled/disabled) of the thermal zone. - - "enabled" means the kernel thermal management is enabled. - - "disabled" will prevent kernel thermal driver action upon trip points - so that user applications can take charge of thermal management. - .set_mode: set the mode (enabled/disabled) of the thermal zone. - .get_trip_type: get the type of certain trip point. - .get_trip_temp: get the temperature above which the certain trip point + .get_mode: + get the current mode (enabled/disabled) of the thermal zone. + + - "enabled" means the kernel thermal management is + enabled. + - "disabled" will prevent kernel thermal driver action + upon trip points so that user applications can take + charge of thermal management. + .set_mode: + set the mode (enabled/disabled) of the thermal zone. + .get_trip_type: + get the type of certain trip point. + .get_trip_temp: + get the temperature above which the certain trip point will be fired. - .set_emul_temp: set the emulation temperature which helps in debugging + .set_emul_temp: + set the emulation temperature which helps in debugging different threshold temperature points. - tzp: thermal zone platform parameters. - passive_delay: number of milliseconds to wait between polls when + tzp: + thermal zone platform parameters. + passive_delay: + number of milliseconds to wait between polls when performing passive cooling. - polling_delay: number of milliseconds to wait between polls when checking + polling_delay: + number of milliseconds to wait between polls when checking whether trip points have been crossed (0 for interrupt driven systems). + :: -1.1.2 void thermal_zone_device_unregister(struct thermal_zone_device *tz) + void thermal_zone_device_unregister(struct thermal_zone_device *tz) This interface function removes the thermal zone device. It deletes the corresponding entry from /sys/class/thermal folder and unbinds all the thermal cooling devices it uses. -1.1.3 struct thermal_zone_device *thermal_zone_of_sensor_register( - struct device *dev, int sensor_id, void *data, - const struct thermal_zone_of_device_ops *ops) + :: + + struct thermal_zone_device + *thermal_zone_of_sensor_register(struct device *dev, int sensor_id, + void *data, + const struct thermal_zone_of_device_ops *ops) This interface adds a new sensor to a DT thermal zone. This function will search the list of thermal zones described in @@ -87,25 +120,33 @@ temperature) and throttle appropriate devices. thermal zone device. The parameters for this interface are: - dev: Device node of sensor containing valid node pointer in + + dev: + Device node of sensor containing valid node pointer in dev->of_node. - sensor_id: a sensor identifier, in case the sensor IP has more + sensor_id: + a sensor identifier, in case the sensor IP has more than one sensors - data: a private pointer (owned by the caller) that will be + data: + a private pointer (owned by the caller) that will be passed back, when a temperature reading is needed. - ops: struct thermal_zone_of_device_ops *. + ops: + `struct thermal_zone_of_device_ops *`. - get_temp: a pointer to a function that reads the + ============== ======================================= + get_temp a pointer to a function that reads the sensor temperature. This is mandatory callback provided by sensor driver. - set_trips: a pointer to a function that sets a + set_trips a pointer to a function that sets a temperature window. When this window is left the driver must inform the thermal core via thermal_zone_device_update. - get_trend: a pointer to a function that reads the + get_trend a pointer to a function that reads the sensor temperature trend. - set_emul_temp: a pointer to a function that sets + set_emul_temp a pointer to a function that sets sensor emulated temperature. + ============== ======================================= + The thermal zone temperature is provided by the get_temp() function pointer of thermal_zone_of_device_ops. When called, it will have the private pointer @data back. @@ -114,8 +155,10 @@ temperature) and throttle appropriate devices. handle. Caller should check the return handle with IS_ERR() for finding whether success or not. -1.1.4 void thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tzd) + :: + + void thermal_zone_of_sensor_unregister(struct device *dev, + struct thermal_zone_device *tzd) This interface unregisters a sensor from a DT thermal zone which was successfully added by interface thermal_zone_of_sensor_register(). @@ -124,21 +167,29 @@ temperature) and throttle appropriate devices. interface. It will also silent the zone by remove the .get_temp() and get_trend() thermal zone device callbacks. -1.1.5 struct thermal_zone_device *devm_thermal_zone_of_sensor_register( - struct device *dev, int sensor_id, - void *data, const struct thermal_zone_of_device_ops *ops) + :: + + struct thermal_zone_device + *devm_thermal_zone_of_sensor_register(struct device *dev, + int sensor_id, + void *data, + const struct thermal_zone_of_device_ops *ops) This interface is resource managed version of thermal_zone_of_sensor_register(). + All details of thermal_zone_of_sensor_register() described in section 1.1.3 is applicable here. + The benefit of using this interface to register sensor is that it is not require to explicitly call thermal_zone_of_sensor_unregister() in error path or during driver unbinding as this is done by driver resource manager. -1.1.6 void devm_thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tzd) + :: + + void devm_thermal_zone_of_sensor_unregister(struct device *dev, + struct thermal_zone_device *tzd) This interface is resource managed version of thermal_zone_of_sensor_unregister(). @@ -147,123 +198,186 @@ temperature) and throttle appropriate devices. Normally this function will not need to be called and the resource management code will ensure that the resource is freed. -1.1.7 int thermal_zone_get_slope(struct thermal_zone_device *tz) + :: + + int thermal_zone_get_slope(struct thermal_zone_device *tz) This interface is used to read the slope attribute value for the thermal zone device, which might be useful for platform drivers for temperature calculations. -1.1.8 int thermal_zone_get_offset(struct thermal_zone_device *tz) + :: + + int thermal_zone_get_offset(struct thermal_zone_device *tz) This interface is used to read the offset attribute value for the thermal zone device, which might be useful for platform drivers for temperature calculations. 1.2 thermal cooling device interface -1.2.1 struct thermal_cooling_device *thermal_cooling_device_register(char *name, - void *devdata, struct thermal_cooling_device_ops *) +------------------------------------ + + + :: + + struct thermal_cooling_device + *thermal_cooling_device_register(char *name, + void *devdata, struct thermal_cooling_device_ops *) This interface function adds a new thermal cooling device (fan/processor/...) - to /sys/class/thermal/ folder as cooling_device[0-*]. It tries to bind itself + to /sys/class/thermal/ folder as `cooling_device[0-*]`. It tries to bind itself to all the thermal zone devices registered at the same time. - name: the cooling device name. - devdata: device private data. - ops: thermal cooling devices call-backs. - .get_max_state: get the Maximum throttle state of the cooling device. - .get_cur_state: get the Currently requested throttle state of the cooling device. - .set_cur_state: set the Current throttle state of the cooling device. -1.2.2 void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev) + name: + the cooling device name. + devdata: + device private data. + ops: + thermal cooling devices call-backs. + + .get_max_state: + get the Maximum throttle state of the cooling device. + .get_cur_state: + get the Currently requested throttle state of the + cooling device. + .set_cur_state: + set the Current throttle state of the cooling device. + + :: + + void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev) This interface function removes the thermal cooling device. It deletes the corresponding entry from /sys/class/thermal folder and unbinds itself from all the thermal zone devices using it. 1.3 interface for binding a thermal zone device with a thermal cooling device -1.3.1 int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz, - int trip, struct thermal_cooling_device *cdev, - unsigned long upper, unsigned long lower, unsigned int weight); +----------------------------------------------------------------------------- + + :: + + int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz, + int trip, struct thermal_cooling_device *cdev, + unsigned long upper, unsigned long lower, unsigned int weight); This interface function binds a thermal cooling device to a particular trip point of a thermal zone device. + This function is usually called in the thermal zone device .bind callback. - tz: the thermal zone device - cdev: thermal cooling device - trip: indicates which trip point in this thermal zone the cooling device - is associated with. - upper:the Maximum cooling state for this trip point. - THERMAL_NO_LIMIT means no upper limit, + + tz: + the thermal zone device + cdev: + thermal cooling device + trip: + indicates which trip point in this thermal zone the cooling device + is associated with. + upper: + the Maximum cooling state for this trip point. + THERMAL_NO_LIMIT means no upper limit, and the cooling device can be in max_state. - lower:the Minimum cooling state can be used for this trip point. - THERMAL_NO_LIMIT means no lower limit, + lower: + the Minimum cooling state can be used for this trip point. + THERMAL_NO_LIMIT means no lower limit, and the cooling device can be in cooling state 0. - weight: the influence of this cooling device in this thermal - zone. See 1.4.1 below for more information. + weight: + the influence of this cooling device in this thermal + zone. See 1.4.1 below for more information. -1.3.2 int thermal_zone_unbind_cooling_device(struct thermal_zone_device *tz, - int trip, struct thermal_cooling_device *cdev); + :: + + int thermal_zone_unbind_cooling_device(struct thermal_zone_device *tz, + int trip, struct thermal_cooling_device *cdev); This interface function unbinds a thermal cooling device from a particular trip point of a thermal zone device. This function is usually called in the thermal zone device .unbind callback. - tz: the thermal zone device - cdev: thermal cooling device - trip: indicates which trip point in this thermal zone the cooling device - is associated with. + + tz: + the thermal zone device + cdev: + thermal cooling device + trip: + indicates which trip point in this thermal zone the cooling device + is associated with. 1.4 Thermal Zone Parameters -1.4.1 struct thermal_bind_params +--------------------------- + + :: + + struct thermal_bind_params + This structure defines the following parameters that are used to bind a zone with a cooling device for a particular trip point. - .cdev: The cooling device pointer - .weight: The 'influence' of a particular cooling device on this - zone. This is relative to the rest of the cooling - devices. For example, if all cooling devices have a - weight of 1, then they all contribute the same. You can - use percentages if you want, but it's not mandatory. A - weight of 0 means that this cooling device doesn't - contribute to the cooling of this zone unless all cooling - devices have a weight of 0. If all weights are 0, then - they all contribute the same. - .trip_mask:This is a bit mask that gives the binding relation between - this thermal zone and cdev, for a particular trip point. - If nth bit is set, then the cdev and thermal zone are bound - for trip point n. - .binding_limits: This is an array of cooling state limits. Must have - exactly 2 * thermal_zone.number_of_trip_points. It is an - array consisting of tuples <lower-state upper-state> of - state limits. Each trip will be associated with one state - limit tuple when binding. A NULL pointer means - <THERMAL_NO_LIMITS THERMAL_NO_LIMITS> on all trips. - These limits are used when binding a cdev to a trip point. - .match: This call back returns success(0) if the 'tz and cdev' need to + + .cdev: + The cooling device pointer + .weight: + The 'influence' of a particular cooling device on this + zone. This is relative to the rest of the cooling + devices. For example, if all cooling devices have a + weight of 1, then they all contribute the same. You can + use percentages if you want, but it's not mandatory. A + weight of 0 means that this cooling device doesn't + contribute to the cooling of this zone unless all cooling + devices have a weight of 0. If all weights are 0, then + they all contribute the same. + .trip_mask: + This is a bit mask that gives the binding relation between + this thermal zone and cdev, for a particular trip point. + If nth bit is set, then the cdev and thermal zone are bound + for trip point n. + .binding_limits: + This is an array of cooling state limits. Must have + exactly 2 * thermal_zone.number_of_trip_points. It is an + array consisting of tuples <lower-state upper-state> of + state limits. Each trip will be associated with one state + limit tuple when binding. A NULL pointer means + <THERMAL_NO_LIMITS THERMAL_NO_LIMITS> on all trips. + These limits are used when binding a cdev to a trip point. + .match: + This call back returns success(0) if the 'tz and cdev' need to be bound, as per platform data. -1.4.2 struct thermal_zone_params + + :: + + struct thermal_zone_params + This structure defines the platform level parameters for a thermal zone. This data, for each thermal zone should come from the platform layer. This is an optional feature where some platforms can choose not to provide this data. - .governor_name: Name of the thermal governor used for this zone - .no_hwmon: a boolean to indicate if the thermal to hwmon sysfs interface - is required. when no_hwmon == false, a hwmon sysfs interface - will be created. when no_hwmon == true, nothing will be done. - In case the thermal_zone_params is NULL, the hwmon interface - will be created (for backward compatibility). - .num_tbps: Number of thermal_bind_params entries for this zone - .tbp: thermal_bind_params entries + + .governor_name: + Name of the thermal governor used for this zone + .no_hwmon: + a boolean to indicate if the thermal to hwmon sysfs interface + is required. when no_hwmon == false, a hwmon sysfs interface + will be created. when no_hwmon == true, nothing will be done. + In case the thermal_zone_params is NULL, the hwmon interface + will be created (for backward compatibility). + .num_tbps: + Number of thermal_bind_params entries for this zone + .tbp: + thermal_bind_params entries 2. sysfs attributes structure +============================= +== ================ RO read only value WO write only value RW read/write value +== ================ Thermal sysfs attributes will be represented under /sys/class/thermal. Hwmon sysfs I/F extension is also available under /sys/class/hwmon if hwmon is compiled in or built as a module. -Thermal zone device sys I/F, created once it's registered: -/sys/class/thermal/thermal_zone[0-*]: +Thermal zone device sys I/F, created once it's registered:: + + /sys/class/thermal/thermal_zone[0-*]: |---type: Type of the thermal zone |---temp: Current temperature |---mode: Working mode of the thermal zone @@ -282,8 +396,9 @@ Thermal zone device sys I/F, created once it's registered: |---slope: Slope constant applied as linear extrapolation |---offset: Offset constant applied as linear extrapolation -Thermal cooling device sys I/F, created once it's registered: -/sys/class/thermal/cooling_device[0-*]: +Thermal cooling device sys I/F, created once it's registered:: + + /sys/class/thermal/cooling_device[0-*]: |---type: Type of the cooling device(processor/fan/...) |---max_state: Maximum cooling state of the cooling device |---cur_state: Current cooling state of the cooling device @@ -299,11 +414,13 @@ the relationship between a thermal zone and its associated cooling device. They are created/removed for each successful execution of thermal_zone_bind_cooling_device/thermal_zone_unbind_cooling_device. -/sys/class/thermal/thermal_zone[0-*]: +:: + + /sys/class/thermal/thermal_zone[0-*]: |---cdev[0-*]: [0-*]th cooling device in current thermal zone |---cdev[0-*]_trip_point: Trip point that cdev[0-*] is associated with |---cdev[0-*]_weight: Influence of the cooling device in - this thermal zone + this thermal zone Besides the thermal zone device sysfs I/F and cooling device sysfs I/F, the generic thermal driver also creates a hwmon sysfs I/F for each _type_ @@ -311,16 +428,17 @@ of thermal zone device. E.g. the generic thermal driver registers one hwmon class device and build the associated hwmon sysfs I/F for all the registered ACPI thermal zones. -/sys/class/hwmon/hwmon[0-*]: +:: + + /sys/class/hwmon/hwmon[0-*]: |---name: The type of the thermal zone devices |---temp[1-*]_input: The current temperature of thermal zone [1-*] |---temp[1-*]_critical: The critical trip point of thermal zone [1-*] Please read Documentation/hwmon/sysfs-interface.rst for additional information. -*************************** -* Thermal zone attributes * -*************************** +Thermal zone attributes +----------------------- type Strings which represent the thermal zone type. @@ -340,54 +458,67 @@ mode This file gives information about the algorithm that is currently managing the thermal zone. It can be either default kernel based algorithm or user space application. - enabled = enable Kernel Thermal management. - disabled = Preventing kernel thermal zone driver actions upon + + enabled + enable Kernel Thermal management. + disabled + Preventing kernel thermal zone driver actions upon trip points so that user application can take full charge of the thermal management. + RW, Optional policy One of the various thermal governors used for a particular zone. + RW, Required available_policies Available thermal governors which can be used for a particular zone. + RO, Required -trip_point_[0-*]_temp +`trip_point_[0-*]_temp` The temperature above which trip point will be fired. + Unit: millidegree Celsius + RO, Optional -trip_point_[0-*]_type +`trip_point_[0-*]_type` Strings which indicate the type of the trip point. - E.g. it can be one of critical, hot, passive, active[0-*] for ACPI + + E.g. it can be one of critical, hot, passive, `active[0-*]` for ACPI thermal zone. + RO, Optional -trip_point_[0-*]_hyst +`trip_point_[0-*]_hyst` The hysteresis value for a trip point, represented as an integer Unit: Celsius RW, Optional -cdev[0-*] +`cdev[0-*]` Sysfs link to the thermal cooling device node where the sys I/F for cooling device throttling control represents. + RO, Optional -cdev[0-*]_trip_point - The trip point in this thermal zone which cdev[0-*] is associated +`cdev[0-*]_trip_point` + The trip point in this thermal zone which `cdev[0-*]` is associated with; -1 means the cooling device is not associated with any trip point. + RO, Optional -cdev[0-*]_weight - The influence of cdev[0-*] in this thermal zone. This value - is relative to the rest of cooling devices in the thermal - zone. For example, if a cooling device has a weight double - than that of other, it's twice as effective in cooling the - thermal zone. - RW, Optional +`cdev[0-*]_weight` + The influence of `cdev[0-*]` in this thermal zone. This value + is relative to the rest of cooling devices in the thermal + zone. For example, if a cooling device has a weight double + than that of other, it's twice as effective in cooling the + thermal zone. + + RW, Optional passive Attribute is only present for zones in which the passive cooling @@ -395,8 +526,11 @@ passive and can be set to a temperature (in millidegrees) to enable a passive trip point for the zone. Activation is done by polling with an interval of 1 second. + Unit: millidegrees Celsius + Valid values: 0 (disabled) or greater than 1000 + RW, Optional emul_temp @@ -407,17 +541,21 @@ emul_temp threshold and its associated cooling action. This is write only node and writing 0 on this node should disable emulation. Unit: millidegree Celsius + WO, Optional - WARNING: Be careful while enabling this option on production systems, - because userland can easily disable the thermal policy by simply - flooding this sysfs node with low temperature values. + WARNING: + Be careful while enabling this option on production systems, + because userland can easily disable the thermal policy by simply + flooding this sysfs node with low temperature values. sustainable_power An estimate of the sustained power that can be dissipated by the thermal zone. Used by the power allocator governor. For - more information see Documentation/thermal/power_allocator.txt + more information see Documentation/thermal/power_allocator.rst + Unit: milliwatts + RW, Optional k_po @@ -425,7 +563,8 @@ k_po controller during temperature overshoot. Temperature overshoot is when the current temperature is above the "desired temperature" trip point. For more information see - Documentation/thermal/power_allocator.txt + Documentation/thermal/power_allocator.rst + RW, Optional k_pu @@ -433,20 +572,23 @@ k_pu controller during temperature undershoot. Temperature undershoot is when the current temperature is below the "desired temperature" trip point. For more information see - Documentation/thermal/power_allocator.txt + Documentation/thermal/power_allocator.rst + RW, Optional k_i The integral term of the power allocator governor's PID controller. This term allows the PID controller to compensate for long term drift. For more information see - Documentation/thermal/power_allocator.txt + Documentation/thermal/power_allocator.rst + RW, Optional k_d The derivative term of the power allocator governor's PID controller. For more information see - Documentation/thermal/power_allocator.txt + Documentation/thermal/power_allocator.rst + RW, Optional integral_cutoff @@ -456,8 +598,10 @@ integral_cutoff example, if integral_cutoff is 0, then the integral term only accumulates error when temperature is above the desired temperature trip point. For more information see - Documentation/thermal/power_allocator.txt + Documentation/thermal/power_allocator.rst + Unit: millidegree Celsius + RW, Optional slope @@ -465,6 +609,7 @@ slope to determine a hotspot temperature based off the sensor's raw readings. It is up to the device driver to determine the usage of these values. + RW, Optional offset @@ -472,28 +617,33 @@ offset to determine a hotspot temperature based off the sensor's raw readings. It is up to the device driver to determine the usage of these values. + RW, Optional -***************************** -* Cooling device attributes * -***************************** +Cooling device attributes +------------------------- type String which represents the type of device, e.g: + - for generic ACPI: should be "Fan", "Processor" or "LCD" - for memory controller device on intel_menlow platform: should be "Memory controller". + RO, Required max_state The maximum permissible cooling state of this cooling device. + RO, Required cur_state The current cooling state of this cooling device. The value can any integer numbers between 0 and max_state: + - cur_state == 0 means no cooling - cur_state == max_state means the maximum cooling. + RW, Required stats/reset @@ -508,9 +658,11 @@ stats/time_in_state_ms: units here is 10mS (similar to other time exported in /proc). RO, Required + stats/total_trans: A single positive value showing the total number of times the state of a cooling device is changed. + RO, Required stats/trans_table: @@ -522,6 +674,7 @@ stats/trans_table: RO, Required 3. A simple implementation +========================== ACPI thermal zone may support multiple trip points like critical, hot, passive, active. If an ACPI thermal zone supports critical, passive, @@ -532,11 +685,10 @@ thermal_cooling_device. Both are considered to have the same effectiveness in cooling the thermal zone. If the processor is listed in _PSL method, and the fan is listed in _AL0 -method, the sys I/F structure will be built like this: +method, the sys I/F structure will be built like this:: -/sys/class/thermal: - -|thermal_zone1: + /sys/class/thermal: + |thermal_zone1: |---type: acpitz |---temp: 37000 |---mode: enabled @@ -557,24 +709,24 @@ method, the sys I/F structure will be built like this: |---cdev1_trip_point: 2 /* cdev1 can be used for active[0]*/ |---cdev1_weight: 1024 -|cooling_device0: + |cooling_device0: |---type: Processor |---max_state: 8 |---cur_state: 0 -|cooling_device3: + |cooling_device3: |---type: Fan |---max_state: 2 |---cur_state: 0 -/sys/class/hwmon: - -|hwmon0: + /sys/class/hwmon: + |hwmon0: |---name: acpitz |---temp1_input: 37000 |---temp1_crit: 100000 4. Event Notification +===================== The framework includes a simple notification mechanism, in the form of a netlink event. Netlink socket initialization is done during the _init_ @@ -587,21 +739,28 @@ event will be one of:{THERMAL_AUX0, THERMAL_AUX1, THERMAL_CRITICAL, THERMAL_DEV_FAULT}. Notification can be sent when the current temperature crosses any of the configured thresholds. -5. Export Symbol APIs: +5. Export Symbol APIs +===================== + +5.1. get_tz_trend +----------------- -5.1: get_tz_trend: This function returns the trend of a thermal zone, i.e the rate of change of temperature of the thermal zone. Ideally, the thermal sensor drivers are supposed to implement the callback. If they don't, the thermal framework calculated the trend by comparing the previous and the current temperature values. -5.2:get_thermal_instance: +5.2. get_thermal_instance +------------------------- + This function returns the thermal_instance corresponding to a given {thermal_zone, cooling_device, trip_point} combination. Returns NULL if such an instance does not exist. -5.3:thermal_notify_framework: +5.3. thermal_notify_framework +----------------------------- + This function handles the trip events from sensor drivers. It starts throttling the cooling devices according to the policy configured. For CRITICAL and HOT trip points, this notifies the respective drivers, @@ -609,12 +768,15 @@ and does actual throttling for other trip points i.e ACTIVE and PASSIVE. The throttling policy is based on the configured platform data; if no platform data is provided, this uses the step_wise throttling policy. -5.4:thermal_cdev_update: +5.4. thermal_cdev_update +------------------------ + This function serves as an arbitrator to set the state of a cooling device. It sets the cooling device to the deepest cooling state if possible. -6. thermal_emergency_poweroff: +6. thermal_emergency_poweroff +============================= On an event of critical trip temperature crossing. Thermal framework allows the system to shutdown gracefully by calling orderly_poweroff(). diff --git a/Documentation/thermal/x86_pkg_temperature_thermal b/Documentation/thermal/x86_pkg_temperature_thermal.rst index 17a3a4c0a0ca..f134dbd3f5a9 100644 --- a/Documentation/thermal/x86_pkg_temperature_thermal +++ b/Documentation/thermal/x86_pkg_temperature_thermal.rst @@ -1,19 +1,23 @@ +=================================== Kernel driver: x86_pkg_temp_thermal -=================== +=================================== Supported chips: + * x86: with package level thermal management + (Verify using: CPUID.06H:EAX[bit 6] =1) Authors: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com> Reference ---- +--------- + Intel® 64 and IA-32 Architectures Software Developer’s Manual (Jan, 2013): Chapter 14.6: PACKAGE LEVEL THERMAL MANAGEMENT Description ---------- +----------- This driver register CPU digital temperature package level sensor as a thermal zone with maximum two user mode configurable trip points. Number of trip points @@ -25,23 +29,27 @@ take any action to control temperature. Threshold management -------------------- Each package will register as a thermal zone under /sys/class/thermal. -Example: -/sys/class/thermal/thermal_zone1 + +Example:: + + /sys/class/thermal/thermal_zone1 This contains two trip points: + - trip_point_0_temp - trip_point_1_temp User can set any temperature between 0 to TJ-Max temperature. Temperature units -are in milli-degree Celsius. Refer to "Documentation/thermal/sysfs-api.txt" for +are in milli-degree Celsius. Refer to "Documentation/thermal/sysfs-api.rst" for thermal sys-fs details. Any value other than 0 in these trip points, can trigger thermal notifications. Setting 0, stops sending thermal notifications. -Thermal notifications: To get kobject-uevent notifications, set the thermal zone -policy to "user_space". For example: echo -n "user_space" > policy - - +Thermal notifications: +To get kobject-uevent notifications, set the thermal zone +policy to "user_space". +For example:: + echo -n "user_space" > policy diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst index 7d2b0178d3f3..fbb314bfa112 100644 --- a/Documentation/trace/kprobetrace.rst +++ b/Documentation/trace/kprobetrace.rst @@ -51,15 +51,17 @@ Synopsis of kprobe_events $argN : Fetch the Nth function argument. (N >= 1) (\*1) $retval : Fetch return value.(\*2) $comm : Fetch current task comm. - +|-offs(FETCHARG) : Fetch memory at FETCHARG +|- offs address.(\*3) + +|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*3)(\*4) NAME=FETCHARG : Set NAME as the argument name of FETCHARG. FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types - (x8/x16/x32/x64), "string" and bitfield are supported. + (x8/x16/x32/x64), "string", "ustring" and bitfield + are supported. (\*1) only for the probe on function entry (offs == 0). (\*2) only for return probe. (\*3) this is useful for fetching a field of data structures. + (\*4) "u" means user-space dereference. See :ref:`user_mem_access`. Types ----- @@ -77,7 +79,8 @@ apply it to registers/stack-entries etc. (for example, '$stack1:x8[8]' is wrong, but '+8($stack):x8[8]' is OK.) String type is a special type, which fetches a "null-terminated" string from kernel space. This means it will fail and store NULL if the string container -has been paged out. +has been paged out. "ustring" type is an alternative of string for user-space. +See :ref:`user_mem_access` for more info.. The string array type is a bit different from other types. For other base types, <base-type>[1] is equal to <base-type> (e.g. +0(%di):x32[1] is same as +0(%di):x32.) But string[1] is not equal to string. The string type itself @@ -92,6 +95,25 @@ Symbol type('symbol') is an alias of u32 or u64 type (depends on BITS_PER_LONG) which shows given pointer in "symbol+offset" style. For $comm, the default type is "string"; any other type is invalid. +.. _user_mem_access: +User Memory Access +------------------ +Kprobe events supports user-space memory access. For that purpose, you can use +either user-space dereference syntax or 'ustring' type. + +The user-space dereference syntax allows you to access a field of a data +structure in user-space. This is done by adding the "u" prefix to the +dereference syntax. For example, +u4(%si) means it will read memory from the +address in the register %si offset by 4, and the memory is expected to be in +user-space. You can use this for strings too, e.g. +u0(%si):string will read +a string from the address in the register %si that is expected to be in user- +space. 'ustring' is a shortcut way of performing the same task. That is, ++0(%si):ustring is equivalent to +u0(%si):string. + +Note that kprobe-event provides the user-memory access syntax but it doesn't +use it transparently. This means if you use normal dereference or string type +for user memory, it might fail, and may always fail on some archs. The user +has to carefully check if the target data is in kernel or user space. Per-Probe Event Filtering ------------------------- @@ -124,6 +146,20 @@ You can check the total number of probe hits and probe miss-hits via The first column is event name, the second is the number of probe hits, the third is the number of probe miss-hits. +Kernel Boot Parameter +--------------------- +You can add and enable new kprobe events when booting up the kernel by +"kprobe_event=" parameter. The parameter accepts a semicolon-delimited +kprobe events, which format is similar to the kprobe_events. +The difference is that the probe definition parameters are comma-delimited +instead of space. For example, adding myprobe event on do_sys_open like below + + p:myprobe do_sys_open dfd=%ax filename=%dx flags=%cx mode=+4($stack) + +should be below for kernel boot parameter (just replace spaces with comma) + + p:myprobe,do_sys_open,dfd=%ax,filename=%dx,flags=%cx,mode=+4($stack) + Usage examples -------------- diff --git a/Documentation/trace/uprobetracer.rst b/Documentation/trace/uprobetracer.rst index 0b21305fabdc..6e75a6c5a2c8 100644 --- a/Documentation/trace/uprobetracer.rst +++ b/Documentation/trace/uprobetracer.rst @@ -42,16 +42,18 @@ Synopsis of uprobe_tracer @+OFFSET : Fetch memory at OFFSET (OFFSET from same file as PATH) $stackN : Fetch Nth entry of stack (N >= 0) $stack : Fetch stack address. - $retval : Fetch return value.(*) + $retval : Fetch return value.(\*1) $comm : Fetch current task comm. - +|-offs(FETCHARG) : Fetch memory at FETCHARG +|- offs address.(**) + +|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*2)(\*3) NAME=FETCHARG : Set NAME as the argument name of FETCHARG. FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types (x8/x16/x32/x64), "string" and bitfield are supported. - (*) only for return probe. - (**) this is useful for fetching a field of data structures. + (\*1) only for return probe. + (\*2) this is useful for fetching a field of data structures. + (\*3) Unlike kprobe event, "u" prefix will just be ignored, becuse uprobe + events can access only user-space memory. Types ----- diff --git a/Documentation/watchdog/hpwdt.rst b/Documentation/watchdog/hpwdt.rst index 94a96371113e..c165d92cfd12 100644 --- a/Documentation/watchdog/hpwdt.rst +++ b/Documentation/watchdog/hpwdt.rst @@ -39,6 +39,10 @@ Last reviewed: 08/20/2018 Default value is set when compiling the kernel. If it is set to "Y", then there is no way of disabling the watchdog once it has been started. + kdumptimeout Minimum timeout in seconds to apply upon receipt of an NMI + before calling panic. (-1) disables the watchdog. When value + is > 0, the timer is reprogrammed with the greater of + value or current timeout value. ============ ================================================================ NOTE: diff --git a/Documentation/watchdog/watchdog-parameters.rst b/Documentation/watchdog/watchdog-parameters.rst index b121caae7798..a3985cc5aeda 100644 --- a/Documentation/watchdog/watchdog-parameters.rst +++ b/Documentation/watchdog/watchdog-parameters.rst @@ -13,6 +13,17 @@ modules. ------------------------------------------------- +watchdog core: + open_timeout: + Maximum time, in seconds, for which the watchdog framework will take + care of pinging a running hardware watchdog until userspace opens the + corresponding /dev/watchdogN device. A value of 0 means an infinite + timeout. Setting this to a non-zero value can be useful to ensure that + either userspace comes up properly, or the board gets reset and allows + fallback logic in the bootloader to try something else. + +------------------------------------------------- + acquirewdt: wdt_stop: Acquire WDT 'stop' io port (default 0x43) diff --git a/MAINTAINERS b/MAINTAINERS index c144bd6a432e..e50faa93e170 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3765,7 +3765,7 @@ F: arch/powerpc/platforms/cell/ CEPH COMMON CODE (LIBCEPH) M: Ilya Dryomov <idryomov@gmail.com> -M: "Yan, Zheng" <zyan@redhat.com> +M: Jeff Layton <jlayton@kernel.org> M: Sage Weil <sage@redhat.com> L: ceph-devel@vger.kernel.org W: http://ceph.com/ @@ -3777,7 +3777,7 @@ F: include/linux/ceph/ F: include/linux/crush/ CEPH DISTRIBUTED FILE SYSTEM CLIENT (CEPH) -M: "Yan, Zheng" <zyan@redhat.com> +M: Jeff Layton <jlayton@kernel.org> M: Sage Weil <sage@redhat.com> M: Ilya Dryomov <idryomov@gmail.com> L: ceph-devel@vger.kernel.org @@ -4683,6 +4683,13 @@ L: linux-mtd@lists.infradead.org S: Supported F: drivers/mtd/nand/raw/denali* +DESIGNWARE EDMA CORE IP DRIVER +M: Gustavo Pimentel <gustavo.pimentel@synopsys.com> +L: dmaengine@vger.kernel.org +S: Maintained +F: drivers/dma/dw-edma/ +F: include/linux/dma/edma.h + DESIGNWARE USB2 DRD IP DRIVER M: Minas Harutyunyan <hminas@synopsys.com> L: linux-usb@vger.kernel.org @@ -6314,9 +6321,8 @@ F: Documentation/devicetree/bindings/counter/ftm-quaddec.txt F: drivers/counter/ftm-quaddec.c FLOPPY DRIVER -M: Jiri Kosina <jikos@kernel.org> -T: git git://git.kernel.org/pub/scm/linux/kernel/git/jikos/floppy.git -S: Odd fixes +S: Orphan +L: linux-block@vger.kernel.org F: drivers/block/floppy.c FMC SUBSYSTEM @@ -7955,6 +7961,33 @@ L: linux-mtd@lists.infradead.org S: Maintained F: drivers/mtd/nand/raw/ingenic/ +INGENIC JZ47xx SoCs +M: Paul Cercueil <paul@crapouillou.net> +S: Maintained +F: arch/mips/boot/dts/ingenic/ +F: arch/mips/include/asm/mach-jz4740/ +F: arch/mips/jz4740/ +F: drivers/clk/ingenic/ +F: drivers/dma/dma-jz4780.c +F: drivers/gpu/drm/ingenic/ +F: drivers/i2c/busses/i2c-jz4780.c +F: drivers/iio/adc/ingenic-adc.c +F: drivers/irqchip/irq-ingenic.c +F: drivers/memory/jz4780-nemc.c +F: drivers/mmc/host/jz4740_mmc.c +F: drivers/mtd/nand/raw/ingenic/ +F: drivers/pinctrl/pinctrl-ingenic.c +F: drivers/power/supply/ingenic-battery.c +F: drivers/pwm/pwm-jz4740.c +F: drivers/rtc/rtc-jz4740.c +F: drivers/tty/serial/8250/8250_ingenic.c +F: drivers/usb/musb/jz4740.c +F: drivers/watchdog/jz4740_wdt.c +F: include/dt-bindings/iio/adc/ingenic,adc.h +F: include/linux/mfd/ingenic-tcu.h +F: sound/soc/jz4740/ +F: sound/soc/codecs/jz47* + INOTIFY M: Jan Kara <jack@suse.cz> R: Amir Goldstein <amir73il@gmail.com> @@ -12807,6 +12840,7 @@ F: drivers/base/power/ F: include/linux/pm.h F: include/linux/pm_* F: include/linux/powercap.h +F: include/linux/intel_rapl.h F: drivers/powercap/ F: kernel/configs/nopm.config @@ -13556,9 +13590,11 @@ L: linux-remoteproc@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/ohad/remoteproc.git S: Maintained F: Documentation/devicetree/bindings/remoteproc/ +F: Documentation/ABI/testing/sysfs-class-remoteproc F: Documentation/remoteproc.txt F: drivers/remoteproc/ F: include/linux/remoteproc.h +F: include/linux/remoteproc/ REMOTE PROCESSOR MESSAGING (RPMSG) SUBSYSTEM M: Ohad Ben-Cohen <ohad@wizery.com> @@ -13568,8 +13604,11 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/ohad/rpmsg.git S: Maintained F: drivers/rpmsg/ F: Documentation/rpmsg.txt +F: Documentation/ABI/testing/sysfs-bus-rpmsg F: include/linux/rpmsg.h F: include/linux/rpmsg/ +F: include/uapi/linux/rpmsg.h +F: samples/rpmsg/ RENESAS CLOCK DRIVERS M: Geert Uytterhoeven <geert+renesas@glider.be> @@ -15857,7 +15896,7 @@ M: Viresh Kumar <viresh.kumar@linaro.org> M: Javi Merino <javi.merino@kernel.org> L: linux-pm@vger.kernel.org S: Supported -F: Documentation/thermal/cpu-cooling-api.txt +F: Documentation/thermal/cpu-cooling-api.rst F: drivers/thermal/cpu_cooling.c F: include/linux/cpu_cooling.h @@ -17073,6 +17112,13 @@ S: Maintained F: drivers/virtio/virtio_input.c F: include/uapi/linux/virtio_input.h +VIRTIO IOMMU DRIVER +M: Jean-Philippe Brucker <jean-philippe.brucker@arm.com> +L: virtualization@lists.linux-foundation.org +S: Maintained +F: drivers/iommu/virtio-iommu.c +F: include/uapi/linux/virtio_iommu.h + VIRTUAL BOX GUEST DEVICE DRIVER M: Hans de Goede <hdegoede@redhat.com> M: Arnd Bergmann <arnd@arndb.de> @@ -17605,9 +17651,8 @@ L: linux-xfs@vger.kernel.org W: http://xfs.org/ T: git git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git S: Supported -F: Documentation/filesystems/xfs.txt +F: Documentation/admin-guide/xfs.rst F: Documentation/ABI/testing/sysfs-fs-xfs -F: Documentation/filesystems/xfs.txt F: Documentation/filesystems/xfs-delayed-logging-design.txt F: Documentation/filesystems/xfs-self-describing-metadata.txt F: fs/xfs/ diff --git a/arch/Kconfig b/arch/Kconfig index e8d19c3cb91f..6dd1faab6ccb 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -128,22 +128,6 @@ config UPROBES managed by the kernel and kept transparent to the probed application. ) -config HAVE_64BIT_ALIGNED_ACCESS - def_bool 64BIT && !HAVE_EFFICIENT_UNALIGNED_ACCESS - help - Some architectures require 64 bit accesses to be 64 bit - aligned, which also requires structs containing 64 bit values - to be 64 bit aligned too. This includes some 32 bit - architectures which can do 64 bit accesses, as well as 64 bit - architectures without unaligned access. - - This symbol should be selected by an architecture if 64 bit - accesses are required to be 64 bit aligned in this way even - though it is not a 64 bit architecture. - - See Documentation/unaligned-memory-access.txt for more - information on the topic of unaligned memory accesses. - config HAVE_EFFICIENT_UNALIGNED_ACCESS bool help diff --git a/arch/alpha/include/asm/io.h b/arch/alpha/include/asm/io.h index ccf9d65166bb..af2c0063dc75 100644 --- a/arch/alpha/include/asm/io.h +++ b/arch/alpha/include/asm/io.h @@ -93,11 +93,6 @@ static inline void * phys_to_virt(unsigned long address) #define page_to_phys(page) page_to_pa(page) -static inline dma_addr_t __deprecated isa_page_to_bus(struct page *page) -{ - return page_to_phys(page); -} - /* Maximum PIO space address supported? */ #define IO_SPACE_LIMIT 0xffff diff --git a/arch/arc/boot/dts/haps_hs.dts b/arch/arc/boot/dts/haps_hs.dts index 1ebfa046492b..44bc522fdec8 100644 --- a/arch/arc/boot/dts/haps_hs.dts +++ b/arch/arc/boot/dts/haps_hs.dts @@ -62,5 +62,35 @@ #interrupt-cells = <1>; interrupts = <20>; }; + + virtio0: virtio@f0100000 { + compatible = "virtio,mmio"; + reg = <0xf0100000 0x2000>; + interrupts = <31>; + }; + + virtio1: virtio@f0102000 { + compatible = "virtio,mmio"; + reg = <0xf0102000 0x2000>; + interrupts = <32>; + }; + + virtio2: virtio@f0104000 { + compatible = "virtio,mmio"; + reg = <0xf0104000 0x2000>; + interrupts = <33>; + }; + + virtio3: virtio@f0106000 { + compatible = "virtio,mmio"; + reg = <0xf0106000 0x2000>; + interrupts = <34>; + }; + + virtio4: virtio@f0108000 { + compatible = "virtio,mmio"; + reg = <0xf0108000 0x2000>; + interrupts = <35>; + }; }; }; diff --git a/arch/arc/boot/dts/hsdk.dts b/arch/arc/boot/dts/hsdk.dts index 9a45cb093096..bfc7f5f5d6f2 100644 --- a/arch/arc/boot/dts/hsdk.dts +++ b/arch/arc/boot/dts/hsdk.dts @@ -8,6 +8,7 @@ */ /dts-v1/; +#include <dt-bindings/gpio/gpio.h> #include <dt-bindings/reset/snps,hsdk-reset.h> / { @@ -252,6 +253,19 @@ dma-coherent; }; + spi0: spi@20000 { + compatible = "snps,dw-apb-ssi"; + reg = <0x20000 0x100>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <16>; + num-cs = <2>; + reg-io-width = <4>; + clocks = <&input_clk>; + cs-gpios = <&creg_gpio 0 GPIO_ACTIVE_LOW>, + <&creg_gpio 1 GPIO_ACTIVE_LOW>; + }; + creg_gpio: gpio@14b0 { compatible = "snps,creg-gpio-hsdk"; reg = <0x14b0 0x4>; diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig index b117e6c16d41..436f2135bdc1 100644 --- a/arch/arc/configs/haps_hs_defconfig +++ b/arch/arc/configs/haps_hs_defconfig @@ -35,10 +35,12 @@ CONFIG_INET=y # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set -# CONFIG_BLK_DEV is not set +CONFIG_VIRTIO_BLK=y CONFIG_NETDEVICES=y +CONFIG_VIRTIO_NET=y # CONFIG_NET_VENDOR_ARC is not set # CONFIG_NET_VENDOR_BROADCOM is not set # CONFIG_NET_VENDOR_INTEL is not set @@ -68,6 +70,7 @@ CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_LOGO=y # CONFIG_HID is not set # CONFIG_USB_SUPPORT is not set +CONFIG_VIRTIO_MMIO=y # CONFIG_IOMMU_SUPPORT is not set CONFIG_EXT2_FS=y CONFIG_EXT2_FS_XATTR=y diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig index c8fb5d60c53f..403125d9c9a3 100644 --- a/arch/arc/configs/hsdk_defconfig +++ b/arch/arc/configs/hsdk_defconfig @@ -46,6 +46,9 @@ CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_8250_DW=y CONFIG_SERIAL_OF_PLATFORM=y # CONFIG_HW_RANDOM is not set +CONFIG_SPI=y +CONFIG_SPI_DESIGNWARE=y +CONFIG_SPI_DW_MMIO=y CONFIG_GPIOLIB=y CONFIG_GPIO_SYSFS=y CONFIG_GPIO_DWAPB=y @@ -66,6 +69,8 @@ CONFIG_MMC=y CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_PLTFM=y CONFIG_MMC_DW=y +CONFIG_DMADEVICES=y +CONFIG_DW_AXI_DMAC=y CONFIG_EXT3_FS=y CONFIG_VFAT_FS=y CONFIG_TMPFS=y diff --git a/arch/arc/include/asm/entry-arcv2.h b/arch/arc/include/asm/entry-arcv2.h index 225e7df2d8ed..f5ae394ebe06 100644 --- a/arch/arc/include/asm/entry-arcv2.h +++ b/arch/arc/include/asm/entry-arcv2.h @@ -7,232 +7,251 @@ #include <asm/irqflags-arcv2.h> #include <asm/thread_info.h> /* For THREAD_SIZE */ +/* + * Interrupt/Exception stack layout (pt_regs) for ARCv2 + * (End of struct aligned to end of page [unless nested]) + * + * INTERRUPT EXCEPTION + * + * manual --------------------- manual + * | orig_r0 | + * | event/ECR | + * | bta | + * | user_r25 | + * | gp | + * | fp | + * | sp | + * | r12 | + * | r30 | + * | r58 | + * | r59 | + * hw autosave --------------------- + * optional | r0 | + * | r1 | + * ~ ~ + * | r9 | + * | r10 | + * | r11 | + * | blink | + * | lpe | + * | lps | + * | lpc | + * | ei base | + * | ldi base | + * | jli base | + * --------------------- + * hw autosave | pc / eret | + * mandatory | stat32 / erstatus | + * --------------------- + */ + /*------------------------------------------------------------------------*/ -.macro INTERRUPT_PROLOGUE called_from +.macro INTERRUPT_PROLOGUE - ; Before jumping to Interrupt Vector, hardware micro-ops did following: + ; (A) Before jumping to Interrupt Vector, hardware micro-ops did following: ; 1. SP auto-switched to kernel mode stack - ; 2. STATUS32.Z flag set to U mode at time of interrupt (U:1, K:0) - ; 3. Auto saved: r0-r11, blink, LPE,LPS,LPC, JLI,LDI,EI, PC, STAT32 + ; 2. STATUS32.Z flag set if in U mode at time of interrupt (U:1,K:0) + ; 3. Auto save: (mandatory) Push PC and STAT32 on stack + ; hardware does even if CONFIG_ARC_IRQ_NO_AUTOSAVE + ; 4. Auto save: (optional) r0-r11, blink, LPE,LPS,LPC, JLI,LDI,EI ; - ; Now manually save: r12, sp, fp, gp, r25 + ; (B) Manually saved some regs: r12,r25,r30, sp,fp,gp, ACCL pair #ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE -.ifnc \called_from, exception - st.as r9, [sp, -10] ; save r9 in it's final stack slot - sub sp, sp, 12 ; skip JLI, LDI, EI - - PUSH lp_count - PUSHAX lp_start - PUSHAX lp_end - PUSH blink - - PUSH r11 - PUSH r10 - - sub sp, sp, 4 ; skip r9 - - PUSH r8 - PUSH r7 - PUSH r6 - PUSH r5 - PUSH r4 - PUSH r3 - PUSH r2 - PUSH r1 - PUSH r0 -.endif -#endif + ; carve pt_regs on stack (case #3), PC/STAT32 already on stack + sub sp, sp, SZ_PT_REGS - 8 -#ifdef CONFIG_ARC_HAS_ACCL_REGS - PUSH r59 - PUSH r58 + __SAVE_REGFILE_HARD +#else + ; carve pt_regs on stack (case #4), which grew partially already + sub sp, sp, PT_r0 #endif - PUSH r30 - PUSH r12 + __SAVE_REGFILE_SOFT +.endm + +/*------------------------------------------------------------------------*/ +.macro EXCEPTION_PROLOGUE + + ; (A) Before jumping to Exception Vector, hardware micro-ops did following: + ; 1. SP auto-switched to kernel mode stack + ; 2. STATUS32.Z flag set if in U mode at time of exception (U:1,K:0) + ; + ; (B) Manually save the complete reg file below + + sub sp, sp, SZ_PT_REGS ; carve pt_regs + + ; _HARD saves r10 clobbered by _SOFT as scratch hence comes first + + __SAVE_REGFILE_HARD + __SAVE_REGFILE_SOFT + + st r0, [sp] ; orig_r0 + + lr r10, [eret] + lr r11, [erstatus] + ST2 r10, r11, PT_ret + + lr r10, [ecr] + lr r11, [erbta] + ST2 r10, r11, PT_event + + ; OUTPUT: r10 has ECR expected by EV_Trap +.endm + +/*------------------------------------------------------------------------ + * This macro saves the registers manually which would normally be autosaved + * by hardware on taken interrupts. It is used by + * - exception handlers (which don't have autosave) + * - interrupt autosave disabled due to CONFIG_ARC_IRQ_NO_AUTOSAVE + */ +.macro __SAVE_REGFILE_HARD + + ST2 r0, r1, PT_r0 + ST2 r2, r3, PT_r2 + ST2 r4, r5, PT_r4 + ST2 r6, r7, PT_r6 + ST2 r8, r9, PT_r8 + ST2 r10, r11, PT_r10 + + st blink, [sp, PT_blink] + + lr r10, [lp_end] + lr r11, [lp_start] + ST2 r10, r11, PT_lpe + + st lp_count, [sp, PT_lpc] + + ; skip JLI, LDI, EI for now +.endm + +/*------------------------------------------------------------------------ + * This macros saves a bunch of other registers which can't be autosaved for + * various reasons: + * - r12: the last caller saved scratch reg since hardware saves in pairs so r0-r11 + * - r30: free reg, used by gcc as scratch + * - ACCL/ACCH pair when they exist + */ +.macro __SAVE_REGFILE_SOFT + + ST2 gp, fp, PT_r26 ; gp (r26), fp (r27) + + st r12, [sp, PT_sp + 4] + st r30, [sp, PT_sp + 8] ; Saving pt_regs->sp correctly requires some extra work due to the way ; Auto stack switch works ; - U mode: retrieve it from AUX_USER_SP ; - K mode: add the offset from current SP where H/w starts auto push ; - ; Utilize the fact that Z bit is set if Intr taken in U mode - mov.nz r9, sp - add.nz r9, r9, SZ_PT_REGS - PT_sp - 4 - bnz 1f + ; 1. Utilize the fact that Z bit is set if Intr taken in U mode + ; 2. Upon entry SP is always saved (for any inspection, unwinding etc), + ; but on return, restored only if U mode - lr r9, [AUX_USER_SP] -1: - PUSH r9 ; SP + lr r10, [AUX_USER_SP] ; U mode SP + + ; ISA requires ADD.nz to have same dest and src reg operands + mov.nz r10, sp + add.nz r10, r10, SZ_PT_REGS ; K mode SP - PUSH fp - PUSH gp + st r10, [sp, PT_sp] ; SP (pt_regs->sp) #ifdef CONFIG_ARC_CURR_IN_REG - PUSH r25 ; user_r25 + st r25, [sp, PT_user_r25] GET_CURR_TASK_ON_CPU r25 -#else - sub sp, sp, 4 #endif -.ifnc \called_from, exception - sub sp, sp, 12 ; BTA/ECR/orig_r0 placeholder per pt_regs -.endif +#ifdef CONFIG_ARC_HAS_ACCL_REGS + ST2 r58, r59, PT_sp + 12 +#endif .endm /*------------------------------------------------------------------------*/ -.macro INTERRUPT_EPILOGUE called_from +.macro __RESTORE_REGFILE_SOFT -.ifnc \called_from, exception - add sp, sp, 12 ; skip BTA/ECR/orig_r0 placeholderss -.endif + LD2 gp, fp, PT_r26 ; gp (r26), fp (r27) -#ifdef CONFIG_ARC_CURR_IN_REG - POP r25 -#else - add sp, sp, 4 -#endif + ld r12, [sp, PT_sp + 4] + ld r30, [sp, PT_sp + 8] - POP gp - POP fp - - ; Don't touch AUX_USER_SP if returning to K mode (Z bit set) - ; (Z bit set on K mode is inverse of INTERRUPT_PROLOGUE) - add.z sp, sp, 4 + ; Restore SP (into AUX_USER_SP) only if returning to U mode + ; - for K mode, it will be implicitly restored as stack is unwound + ; - Z flag set on K is inverse of what hardware does on interrupt entry + ; but that doesn't really matter bz 1f - POPAX AUX_USER_SP + ld r10, [sp, PT_sp] ; SP (pt_regs->sp) + sr r10, [AUX_USER_SP] 1: - POP r12 - POP r30 -#ifdef CONFIG_ARC_HAS_ACCL_REGS - POP r58 - POP r59 +#ifdef CONFIG_ARC_CURR_IN_REG + ld r25, [sp, PT_user_r25] #endif -#ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE -.ifnc \called_from, exception - POP r0 - POP r1 - POP r2 - POP r3 - POP r4 - POP r5 - POP r6 - POP r7 - POP r8 - POP r9 - POP r10 - POP r11 - - POP blink - POPAX lp_end - POPAX lp_start - - POP r9 - mov lp_count, r9 - - add sp, sp, 12 ; skip JLI, LDI, EI - ld.as r9, [sp, -10] ; reload r9 which got clobbered -.endif +#ifdef CONFIG_ARC_HAS_ACCL_REGS + LD2 r58, r59, PT_sp + 12 #endif +.endm +/*------------------------------------------------------------------------*/ +.macro __RESTORE_REGFILE_HARD + + ld blink, [sp, PT_blink] + + LD2 r10, r11, PT_lpe + sr r10, [lp_end] + sr r11, [lp_start] + + ld r10, [sp, PT_lpc] ; lp_count can't be target of LD + mov lp_count, r10 + + LD2 r0, r1, PT_r0 + LD2 r2, r3, PT_r2 + LD2 r4, r5, PT_r4 + LD2 r6, r7, PT_r6 + LD2 r8, r9, PT_r8 + LD2 r10, r11, PT_r10 .endm + /*------------------------------------------------------------------------*/ -.macro EXCEPTION_PROLOGUE +.macro INTERRUPT_EPILOGUE - ; Before jumping to Exception Vector, hardware micro-ops did following: - ; 1. SP auto-switched to kernel mode stack - ; 2. STATUS32.Z flag set to U mode at time of interrupt (U:1,K:0) - ; - ; Now manually save the complete reg file - - PUSH r9 ; freeup a register: slot of erstatus - - PUSHAX eret - sub sp, sp, 12 ; skip JLI, LDI, EI - PUSH lp_count - PUSHAX lp_start - PUSHAX lp_end - PUSH blink - - PUSH r11 - PUSH r10 - - ld.as r9, [sp, 10] ; load stashed r9 (status32 stack slot) - lr r10, [erstatus] - st.as r10, [sp, 10] ; save status32 at it's right stack slot - - PUSH r9 - PUSH r8 - PUSH r7 - PUSH r6 - PUSH r5 - PUSH r4 - PUSH r3 - PUSH r2 - PUSH r1 - PUSH r0 - - ; -- for interrupts, regs above are auto-saved by h/w in that order -- - ; Now do what ISR prologue does (manually save r12, sp, fp, gp, r25) - ; - ; Set Z flag if this was from U mode (expected by INTERRUPT_PROLOGUE) - ; Although H/w exception micro-ops do set Z flag for U mode (just like - ; for interrupts), it could get clobbered in case we soft land here from - ; a TLB Miss exception handler (tlbex.S) + ; INPUT: r0 has STAT32 of calling context + ; INPUT: Z flag set if returning to K mode - and r10, r10, STATUS_U_MASK - xor.f 0, r10, STATUS_U_MASK + ; _SOFT clobbers r10 restored by _HARD hence the order - INTERRUPT_PROLOGUE exception + __RESTORE_REGFILE_SOFT - PUSHAX erbta - PUSHAX ecr ; r9 contains ECR, expected by EV_Trap +#ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE + __RESTORE_REGFILE_HARD + add sp, sp, SZ_PT_REGS - 8 +#else + add sp, sp, PT_r0 +#endif - PUSH r0 ; orig_r0 .endm /*------------------------------------------------------------------------*/ .macro EXCEPTION_EPILOGUE - ; Assumes r0 has PT_status32 - btst r0, STATUS_U_BIT ; Z flag set if K, used in INTERRUPT_EPILOGUE - - add sp, sp, 8 ; orig_r0/ECR don't need restoring - POPAX erbta - - INTERRUPT_EPILOGUE exception + ; INPUT: r0 has STAT32 of calling context - POP r0 - POP r1 - POP r2 - POP r3 - POP r4 - POP r5 - POP r6 - POP r7 - POP r8 - POP r9 - POP r10 - POP r11 + btst r0, STATUS_U_BIT ; Z flag set if K, used in restoring SP - POP blink - POPAX lp_end - POPAX lp_start + ld r10, [sp, PT_event + 4] + sr r10, [erbta] - POP r9 - mov lp_count, r9 + LD2 r10, r11, PT_ret + sr r10, [eret] + sr r11, [erstatus] - add sp, sp, 12 ; skip JLI, LDI, EI - POPAX eret - POPAX erstatus + __RESTORE_REGFILE_SOFT + __RESTORE_REGFILE_HARD - ld.as r9, [sp, -12] ; reload r9 which got clobbered + add sp, sp, SZ_PT_REGS .endm .macro FAKE_RET_FROM_EXCPN diff --git a/arch/arc/include/asm/entry-compact.h b/arch/arc/include/asm/entry-compact.h index 66ba1bf21d28..66a292335ee6 100644 --- a/arch/arc/include/asm/entry-compact.h +++ b/arch/arc/include/asm/entry-compact.h @@ -195,8 +195,8 @@ PUSHAX CTOP_AUX_EFLAGS #endif - lr r9, [ecr] - st r9, [sp, PT_event] /* EV_Trap expects r9 to have ECR */ + lr r10, [ecr] + st r10, [sp, PT_event] /* EV_Trap expects r10 to have ECR */ .endm /*-------------------------------------------------------------- diff --git a/arch/arc/include/asm/linkage.h b/arch/arc/include/asm/linkage.h index 54f5ec5c1759..a0eeb9f8f0a9 100644 --- a/arch/arc/include/asm/linkage.h +++ b/arch/arc/include/asm/linkage.h @@ -10,6 +10,24 @@ #ifdef __ASSEMBLY__ +.macro ST2 e, o, off +#ifdef CONFIG_ARC_HAS_LL64 + std \e, [sp, \off] +#else + st \e, [sp, \off] + st \o, [sp, \off+4] +#endif +.endm + +.macro LD2 e, o, off +#ifdef CONFIG_ARC_HAS_LL64 + ldd \e, [sp, \off] +#else + ld \e, [sp, \off] + ld \o, [sp, \off+4] +#endif +.endm + #define ASM_NL ` /* use '`' to mark new line in macro */ /* annotation for data we want in DCCM - if enabled in .config */ diff --git a/arch/arc/include/asm/pgtable.h b/arch/arc/include/asm/pgtable.h index da446180f17b..1d87c18a2976 100644 --- a/arch/arc/include/asm/pgtable.h +++ b/arch/arc/include/asm/pgtable.h @@ -32,7 +32,7 @@ #ifndef _ASM_ARC_PGTABLE_H #define _ASM_ARC_PGTABLE_H -#include <linux/const.h> +#include <linux/bits.h> #define __ARCH_USE_5LEVEL_HACK #include <asm-generic/pgtable-nopmd.h> #include <asm/page.h> @@ -215,11 +215,11 @@ #define BITS_FOR_PTE (PGDIR_SHIFT - PAGE_SHIFT) #define BITS_FOR_PGD (32 - PGDIR_SHIFT) -#define PGDIR_SIZE _BITUL(PGDIR_SHIFT) /* vaddr span, not PDG sz */ +#define PGDIR_SIZE BIT(PGDIR_SHIFT) /* vaddr span, not PDG sz */ #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define PTRS_PER_PTE _BITUL(BITS_FOR_PTE) -#define PTRS_PER_PGD _BITUL(BITS_FOR_PGD) +#define PTRS_PER_PTE BIT(BITS_FOR_PTE) +#define PTRS_PER_PGD BIT(BITS_FOR_PGD) /* * Number of entries a user land program use. diff --git a/arch/arc/kernel/asm-offsets.c b/arch/arc/kernel/asm-offsets.c index dba116535005..1f621e416521 100644 --- a/arch/arc/kernel/asm-offsets.c +++ b/arch/arc/kernel/asm-offsets.c @@ -55,7 +55,14 @@ int main(void) DEFINE(PT_r5, offsetof(struct pt_regs, r5)); DEFINE(PT_r6, offsetof(struct pt_regs, r6)); DEFINE(PT_r7, offsetof(struct pt_regs, r7)); + DEFINE(PT_r8, offsetof(struct pt_regs, r8)); + DEFINE(PT_r10, offsetof(struct pt_regs, r10)); + DEFINE(PT_r26, offsetof(struct pt_regs, r26)); DEFINE(PT_ret, offsetof(struct pt_regs, ret)); + DEFINE(PT_blink, offsetof(struct pt_regs, blink)); + DEFINE(PT_lpe, offsetof(struct pt_regs, lp_end)); + DEFINE(PT_lpc, offsetof(struct pt_regs, lp_count)); + DEFINE(PT_user_r25, offsetof(struct pt_regs, user_r25)); DEFINE(SZ_CALLEE_REGS, sizeof(struct callee_regs)); DEFINE(SZ_PT_REGS, sizeof(struct pt_regs)); diff --git a/arch/arc/kernel/entry-arcv2.S b/arch/arc/kernel/entry-arcv2.S index 14254b866fdc..12d5f12d10d2 100644 --- a/arch/arc/kernel/entry-arcv2.S +++ b/arch/arc/kernel/entry-arcv2.S @@ -67,7 +67,7 @@ reserved: ENTRY(handle_interrupt) - INTERRUPT_PROLOGUE irq + INTERRUPT_PROLOGUE # irq control APIs local_irq_save/restore/disable/enable fiddle with # global interrupt enable bits in STATUS32 (.IE for 1 prio, .E[] for 2 prio) @@ -79,7 +79,7 @@ ENTRY(handle_interrupt) # # Note this disable is only for consistent book-keeping as further interrupts # will be disabled anyways even w/o this. Hardware tracks active interrupts - # seperately in AUX_IRQ_ACTIVE.active and will not take new interrupts + # seperately in AUX_IRQ_ACT.active and will not take new interrupts # unless this one returns (or higher prio becomes pending in 2-prio scheme) IRQ_DISABLE @@ -200,17 +200,18 @@ restore_regs: ld r0, [sp, PT_status32] ; U/K mode at time of entry lr r10, [AUX_IRQ_ACT] - bmsk r11, r10, 15 ; AUX_IRQ_ACT.ACTIVE + bmsk r11, r10, 15 ; extract AUX_IRQ_ACT.active breq r11, 0, .Lexcept_ret ; No intr active, ret from Exception ;####### Return from Intr ####### +.Lisr_ret: + debug_marker_l1: ; bbit1.nt r0, STATUS_DE_BIT, .Lintr_ret_to_delay_slot btst r0, STATUS_DE_BIT ; Z flag set if bit clear bnz .Lintr_ret_to_delay_slot ; branch if STATUS_DE_BIT set -.Lisr_ret_fast_path: ; Handle special case #1: (Entry via Exception, Return via IRQ) ; ; Exception in U mode, preempted in kernel, Intr taken (K mode), orig @@ -223,7 +224,7 @@ debug_marker_l1: bset.nz r11, r11, AUX_IRQ_ACT_BIT_U ; NZ means U sr r11, [AUX_IRQ_ACT] - INTERRUPT_EPILOGUE irq + INTERRUPT_EPILOGUE rtie ;####### Return from Exception / pure kernel mode ####### @@ -244,8 +245,8 @@ debug_marker_syscall: ; ; IRQ RTIE won't reliably restore DE bit and/or BTA, needs workaround ; -; Solution is return from Intr w/o any delay slot quirks into a kernel trampoline -; and from pure kernel mode return to delay slot which handles DS bit/BTA correctly +; Solution is to drop out of interrupt context into pure kernel mode +; and return from pure kernel mode which does right things for delay slot .Lintr_ret_to_delay_slot: debug_marker_ds: @@ -254,48 +255,9 @@ debug_marker_ds: add r2, r2, 1 st r2, [@intr_to_DE_cnt] - ld r2, [sp, PT_ret] - ld r3, [sp, PT_status32] - - ; STAT32 for Int return created from scratch - ; (No delay dlot, disable Further intr in trampoline) - - bic r0, r3, STATUS_U_MASK|STATUS_DE_MASK|STATUS_IE_MASK|STATUS_L_MASK - st r0, [sp, PT_status32] - - mov r1, .Lintr_ret_to_delay_slot_2 - st r1, [sp, PT_ret] - - ; Orig exception PC/STAT32 safekept @orig_r0 and @event stack slots - st r2, [sp, 0] - st r3, [sp, 4] - - b .Lisr_ret_fast_path - -.Lintr_ret_to_delay_slot_2: - ; Trampoline to restore orig exception PC/STAT32/BTA/AUX_USER_SP - sub sp, sp, SZ_PT_REGS - st r9, [sp, -4] - - ld r9, [sp, 0] - sr r9, [eret] - - ld r9, [sp, 4] - sr r9, [erstatus] - - ; restore AUX_USER_SP if returning to U mode - bbit0 r9, STATUS_U_BIT, 1f - ld r9, [sp, PT_sp] - sr r9, [AUX_USER_SP] - -1: - ld r9, [sp, 8] - sr r9, [erbta] - - ld r9, [sp, -4] - add sp, sp, SZ_PT_REGS - - ; return from pure kernel mode to delay slot - rtie + ; drop out of interrupt context (clear AUX_IRQ_ACT.active) + bmskn r11, r10, 15 + sr r11, [AUX_IRQ_ACT] + b .Lexcept_ret END(ret_from_exception) diff --git a/arch/arc/kernel/entry-compact.S b/arch/arc/kernel/entry-compact.S index 7fe59880c16b..5cb0cd7e4eab 100644 --- a/arch/arc/kernel/entry-compact.S +++ b/arch/arc/kernel/entry-compact.S @@ -256,7 +256,7 @@ ENTRY(EV_TLBProtV) EXCEPTION_PROLOGUE - mov r2, r9 ; ECR set into r9 already + mov r2, r10 ; ECR set into r10 already lr r0, [efa] ; Faulting Data address (not part of pt_regs saved above) ; Exception auto-disables further Intr/exceptions. diff --git a/arch/arc/kernel/entry.S b/arch/arc/kernel/entry.S index a2bfacbcfce1..72be01270e24 100644 --- a/arch/arc/kernel/entry.S +++ b/arch/arc/kernel/entry.S @@ -232,8 +232,8 @@ ENTRY(EV_Trap) EXCEPTION_PROLOGUE ;============ TRAP 1 :breakpoints - ; Check ECR for trap with arg (PROLOGUE ensures r9 has ECR) - bmsk.f 0, r9, 7 + ; Check ECR for trap with arg (PROLOGUE ensures r10 has ECR) + bmsk.f 0, r10, 7 bnz trap_with_param ;============ TRAP (no param): syscall top level diff --git a/arch/arc/kernel/unwind.c b/arch/arc/kernel/unwind.c index 182ce67dfe10..c2663fce7f6c 100644 --- a/arch/arc/kernel/unwind.c +++ b/arch/arc/kernel/unwind.c @@ -181,11 +181,6 @@ static void *__init unw_hdr_alloc_early(unsigned long sz) return memblock_alloc_from(sz, sizeof(unsigned int), MAX_DMA_ADDRESS); } -static void *unw_hdr_alloc(unsigned long sz) -{ - return kmalloc(sz, GFP_KERNEL); -} - static void init_unwind_table(struct unwind_table *table, const char *name, const void *core_start, unsigned long core_size, const void *init_start, unsigned long init_size, @@ -366,6 +361,10 @@ ret_err: } #ifdef CONFIG_MODULES +static void *unw_hdr_alloc(unsigned long sz) +{ + return kmalloc(sz, GFP_KERNEL); +} static struct unwind_table *last_table; diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index 81e84426fe21..3861543b66a0 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -63,24 +63,19 @@ void do_page_fault(unsigned long address, struct pt_regs *regs) struct vm_area_struct *vma = NULL; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; - int si_code = SEGV_MAPERR; - int ret; - vm_fault_t fault; - int write = regs->ecr_cause & ECR_C_PROTV_STORE; /* ST/EX */ - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + int sig, si_code = SEGV_MAPERR; + unsigned int write = 0, exec = 0, mask; + vm_fault_t fault = VM_FAULT_SIGSEGV; /* handle_mm_fault() output */ + unsigned int flags; /* handle_mm_fault() input */ /* - * We fault-in kernel-space virtual memory on-demand. The - * 'reference' page table is init_mm.pgd. - * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (address >= VMALLOC_START && !user_mode(regs)) { - ret = handle_kernel_vaddr_fault(address); - if (unlikely(ret)) + if (unlikely(handle_kernel_vaddr_fault(address))) goto no_context; else return; @@ -93,143 +88,117 @@ void do_page_fault(unsigned long address, struct pt_regs *regs) if (faulthandler_disabled() || !mm) goto no_context; + if (regs->ecr_cause & ECR_C_PROTV_STORE) /* ST/EX */ + write = 1; + else if ((regs->ecr_vec == ECR_V_PROTV) && + (regs->ecr_cause == ECR_C_PROTV_INST_FETCH)) + exec = 1; + + flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; if (user_mode(regs)) flags |= FAULT_FLAG_USER; + if (write) + flags |= FAULT_FLAG_WRITE; + retry: down_read(&mm->mmap_sem); + vma = find_vma(mm, address); if (!vma) goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (expand_stack(vma, address)) - goto bad_area; + if (unlikely(address < vma->vm_start)) { + if (!(vma->vm_flags & VM_GROWSDOWN) || expand_stack(vma, address)) + goto bad_area; + } /* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. + * vm_area is good, now check permissions for this memory access */ -good_area: - si_code = SEGV_ACCERR; - - /* Handle protection violation, execute on heap or stack */ - - if ((regs->ecr_vec == ECR_V_PROTV) && - (regs->ecr_cause == ECR_C_PROTV_INST_FETCH)) + mask = VM_READ; + if (write) + mask = VM_WRITE; + if (exec) + mask = VM_EXEC; + + if (!(vma->vm_flags & mask)) { + si_code = SEGV_ACCERR; goto bad_area; - - if (write) { - if (!(vma->vm_flags & VM_WRITE)) - goto bad_area; - flags |= FAULT_FLAG_WRITE; - } else { - if (!(vma->vm_flags & (VM_READ | VM_EXEC))) - goto bad_area; } - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ fault = handle_mm_fault(vma, address, flags); - if (fatal_signal_pending(current)) { + /* + * Fault retry nuances + */ + if (unlikely(fault & VM_FAULT_RETRY)) { /* - * if fault retry, mmap_sem already relinquished by core mm - * so OK to return to user mode (with signal handled first) + * If fault needs to be retried, handle any pending signals + * first (by returning to user mode). + * mmap_sem already relinquished by core mm for RETRY case */ - if (fault & VM_FAULT_RETRY) { + if (fatal_signal_pending(current)) { if (!user_mode(regs)) goto no_context; return; } - } - - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); - - if (likely(!(fault & VM_FAULT_ERROR))) { + /* + * retry state machine + */ if (flags & FAULT_FLAG_ALLOW_RETRY) { - /* To avoid updating stats twice for retry case */ - if (fault & VM_FAULT_MAJOR) { - tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, - regs, address); - } else { - tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, - regs, address); - } - - if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; - flags |= FAULT_FLAG_TRIED; - goto retry; - } + flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; + goto retry; } - - /* Fault Handled Gracefully */ - up_read(&mm->mmap_sem); - return; } - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGSEGV) - goto bad_area; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - - /* no man's land */ - BUG(); +bad_area: + up_read(&mm->mmap_sem); /* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. + * Major/minor page fault accounting + * (in case of retry we only land here once) */ -bad_area: - up_read(&mm->mmap_sem); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); - /* User mode accesses just cause a SIGSEGV */ - if (user_mode(regs)) { - tsk->thread.fault_address = address; - force_sig_fault(SIGSEGV, si_code, (void __user *)address); - return; - } + if (likely(!(fault & VM_FAULT_ERROR))) { + if (fault & VM_FAULT_MAJOR) { + tsk->maj_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, + regs, address); + } else { + tsk->min_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, + regs, address); + } -no_context: - /* Are we prepared to handle this kernel fault? - * - * (The kernel has valid exception-points in the source - * when it accesses user-memory. When it fails in one - * of those points, we find it in a table and do a jump - * to some fixup code that loads an appropriate error - * code) - */ - if (fixup_exception(regs)) + /* Normal return path: fault Handled Gracefully */ return; + } - die("Oops", regs, address); - -out_of_memory: - up_read(&mm->mmap_sem); + if (!user_mode(regs)) + goto no_context; - if (user_mode(regs)) { + if (fault & VM_FAULT_OOM) { pagefault_out_of_memory(); return; } - goto no_context; + if (fault & VM_FAULT_SIGBUS) { + sig = SIGBUS; + si_code = BUS_ADRERR; + } + else { + sig = SIGSEGV; + } -do_sigbus: - up_read(&mm->mmap_sem); + tsk->thread.fault_address = address; + force_sig_fault(sig, si_code, (void __user *)address); + return; - if (!user_mode(regs)) - goto no_context; +no_context: + if (fixup_exception(regs)) + return; - tsk->thread.fault_address = address; - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); + die("Oops", regs, address); } diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S index 471a97bf492d..c55d95dd2f39 100644 --- a/arch/arc/mm/tlbex.S +++ b/arch/arc/mm/tlbex.S @@ -393,6 +393,17 @@ EV_TLBMissD_fast_ret: ; additional label for VDK OS-kit instrumentation ;-------- Common routine to call Linux Page Fault Handler ----------- do_slow_path_pf: +#ifdef CONFIG_ISA_ARCV2 + ; Set Z flag if exception in U mode. Hardware micro-ops do this on any + ; taken interrupt/exception, and thus is already the case at the entry + ; above, but ensuing code would have already clobbered. + ; EXCEPTION_PROLOGUE called in slow path, relies on correct Z flag set + + lr r2, [erstatus] + and r2, r2, STATUS_U_MASK + bxor.f 0, r2, STATUS_U_BIT +#endif + ; Restore the 4-scratch regs saved by fast path miss handler TLBMISS_RESTORE_REGS diff --git a/arch/arc/plat-eznps/include/plat/ctop.h b/arch/arc/plat-eznps/include/plat/ctop.h index 309a994f64f0..a4a61531c7fb 100644 --- a/arch/arc/plat-eznps/include/plat/ctop.h +++ b/arch/arc/plat-eznps/include/plat/ctop.h @@ -10,6 +10,7 @@ #error "Incorrect ctop.h include" #endif +#include <linux/bits.h> #include <linux/types.h> #include <soc/nps/common.h> @@ -51,19 +52,19 @@ #define CTOP_INST_AXOR_DI_R2_R2_R3 0x4A664C06 /* Do not use D$ for address in 2G-3G */ -#define HW_COMPLY_KRN_NOT_D_CACHED _BITUL(28) +#define HW_COMPLY_KRN_NOT_D_CACHED BIT(28) #define NPS_MSU_EN_CFG 0x80 #define NPS_CRG_BLKID 0x480 -#define NPS_CRG_SYNC_BIT _BITUL(0) +#define NPS_CRG_SYNC_BIT BIT(0) #define NPS_GIM_BLKID 0x5C0 /* GIM registers and fields*/ -#define NPS_GIM_UART_LINE _BITUL(7) -#define NPS_GIM_DBG_LAN_EAST_TX_DONE_LINE _BITUL(10) -#define NPS_GIM_DBG_LAN_EAST_RX_RDY_LINE _BITUL(11) -#define NPS_GIM_DBG_LAN_WEST_TX_DONE_LINE _BITUL(25) -#define NPS_GIM_DBG_LAN_WEST_RX_RDY_LINE _BITUL(26) +#define NPS_GIM_UART_LINE BIT(7) +#define NPS_GIM_DBG_LAN_EAST_TX_DONE_LINE BIT(10) +#define NPS_GIM_DBG_LAN_EAST_RX_RDY_LINE BIT(11) +#define NPS_GIM_DBG_LAN_WEST_TX_DONE_LINE BIT(25) +#define NPS_GIM_DBG_LAN_WEST_RX_RDY_LINE BIT(26) #ifndef __ASSEMBLY__ /* Functional registers definition */ diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h index f11c35cf0b74..7a0596fcb2e7 100644 --- a/arch/arm/include/asm/io.h +++ b/arch/arm/include/asm/io.h @@ -30,7 +30,6 @@ * ISA I/O bus memory addresses are 1:1 with the physical address. */ #define isa_virt_to_bus virt_to_phys -#define isa_page_to_bus page_to_phys #define isa_bus_to_virt phys_to_virt /* diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index b3d439c41c7b..deef17f34bd2 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -55,6 +55,13 @@ void *module_alloc(unsigned long size) } #endif +bool module_exit_section(const char *name) +{ + return strstarts(name, ".exit") || + strstarts(name, ".ARM.extab.exit") || + strstarts(name, ".ARM.exidx.exit"); +} + int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relindex, struct module *module) diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 0e417233dad7..890eeaac3cbb 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -27,28 +27,6 @@ #ifdef CONFIG_MMU -#ifdef CONFIG_KPROBES -static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr) -{ - int ret = 0; - - if (!user_mode(regs)) { - /* kprobe_running() needs smp_processor_id() */ - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, fsr)) - ret = 1; - preempt_enable(); - } - - return ret; -} -#else -static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr) -{ - return 0; -} -#endif - /* * This is useful to dump out the page tables associated with * 'addr' in mm 'mm'. @@ -265,7 +243,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) vm_fault_t fault; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; - if (notify_page_fault(regs, fsr)) + if (kprobe_page_fault(regs, fsr)) return 0; tsk = current; diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 86f81b5afd95..e1ea69994e0f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -24,6 +24,7 @@ config ARM64 select ARCH_HAS_KCOV select ARCH_HAS_KEEPINITRD select ARCH_HAS_MEMBARRIER_SYNC_CORE + select ARCH_HAS_PTE_DEVMAP select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SETUP_DMA_OPS select ARCH_HAS_SET_DIRECT_MAP diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index f318258a14be..92d2e9f28f28 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -16,6 +16,7 @@ #define PTE_WRITE (PTE_DBM) /* same as DBM (51) */ #define PTE_DIRTY (_AT(pteval_t, 1) << 55) #define PTE_SPECIAL (_AT(pteval_t, 1) << 56) +#define PTE_DEVMAP (_AT(pteval_t, 1) << 57) #define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */ #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 3052381baaeb..87a4b2ddc1a1 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -79,6 +79,7 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) #define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN)) #define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) +#define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP)) #define pte_cont_addr_end(addr, end) \ ({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \ @@ -206,6 +207,11 @@ static inline pmd_t pmd_mkcont(pmd_t pmd) return __pmd(pmd_val(pmd) | PMD_SECT_CONT); } +static inline pte_t pte_mkdevmap(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_DEVMAP)); +} + static inline void set_pte(pte_t *ptep, pte_t pte) { WRITE_ONCE(*ptep, pte); @@ -388,6 +394,11 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT)) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define pmd_devmap(pmd) pte_devmap(pmd_pte(pmd)) +#endif +#define pmd_mkdevmap(pmd) pte_pmd(pte_mkdevmap(pmd_pte(pmd))) + #define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd)) #define __phys_to_pmd_val(phys) __phys_to_pte_val(phys) #define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT) @@ -673,6 +684,16 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, { return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty); } + +static inline int pud_devmap(pud_t pud) +{ + return 0; +} + +static inline int pgd_devmap(pgd_t pgd) +{ + return 0; +} #endif /* diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index a7522fca1105..06ebcfef73df 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -9,7 +9,7 @@ #ifndef __ASM_SYSREG_H #define __ASM_SYSREG_H -#include <linux/const.h> +#include <linux/bits.h> #include <linux/stringify.h> /* @@ -478,31 +478,31 @@ #define SYS_CNTV_CVAL_EL02 sys_reg(3, 5, 14, 3, 2) /* Common SCTLR_ELx flags. */ -#define SCTLR_ELx_DSSBS (_BITUL(44)) -#define SCTLR_ELx_ENIA (_BITUL(31)) -#define SCTLR_ELx_ENIB (_BITUL(30)) -#define SCTLR_ELx_ENDA (_BITUL(27)) -#define SCTLR_ELx_EE (_BITUL(25)) -#define SCTLR_ELx_IESB (_BITUL(21)) -#define SCTLR_ELx_WXN (_BITUL(19)) -#define SCTLR_ELx_ENDB (_BITUL(13)) -#define SCTLR_ELx_I (_BITUL(12)) -#define SCTLR_ELx_SA (_BITUL(3)) -#define SCTLR_ELx_C (_BITUL(2)) -#define SCTLR_ELx_A (_BITUL(1)) -#define SCTLR_ELx_M (_BITUL(0)) +#define SCTLR_ELx_DSSBS (BIT(44)) +#define SCTLR_ELx_ENIA (BIT(31)) +#define SCTLR_ELx_ENIB (BIT(30)) +#define SCTLR_ELx_ENDA (BIT(27)) +#define SCTLR_ELx_EE (BIT(25)) +#define SCTLR_ELx_IESB (BIT(21)) +#define SCTLR_ELx_WXN (BIT(19)) +#define SCTLR_ELx_ENDB (BIT(13)) +#define SCTLR_ELx_I (BIT(12)) +#define SCTLR_ELx_SA (BIT(3)) +#define SCTLR_ELx_C (BIT(2)) +#define SCTLR_ELx_A (BIT(1)) +#define SCTLR_ELx_M (BIT(0)) #define SCTLR_ELx_FLAGS (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \ SCTLR_ELx_SA | SCTLR_ELx_I | SCTLR_ELx_IESB) /* SCTLR_EL2 specific flags. */ -#define SCTLR_EL2_RES1 ((_BITUL(4)) | (_BITUL(5)) | (_BITUL(11)) | (_BITUL(16)) | \ - (_BITUL(18)) | (_BITUL(22)) | (_BITUL(23)) | (_BITUL(28)) | \ - (_BITUL(29))) -#define SCTLR_EL2_RES0 ((_BITUL(6)) | (_BITUL(7)) | (_BITUL(8)) | (_BITUL(9)) | \ - (_BITUL(10)) | (_BITUL(13)) | (_BITUL(14)) | (_BITUL(15)) | \ - (_BITUL(17)) | (_BITUL(20)) | (_BITUL(24)) | (_BITUL(26)) | \ - (_BITUL(27)) | (_BITUL(30)) | (_BITUL(31)) | \ +#define SCTLR_EL2_RES1 ((BIT(4)) | (BIT(5)) | (BIT(11)) | (BIT(16)) | \ + (BIT(18)) | (BIT(22)) | (BIT(23)) | (BIT(28)) | \ + (BIT(29))) +#define SCTLR_EL2_RES0 ((BIT(6)) | (BIT(7)) | (BIT(8)) | (BIT(9)) | \ + (BIT(10)) | (BIT(13)) | (BIT(14)) | (BIT(15)) | \ + (BIT(17)) | (BIT(20)) | (BIT(24)) | (BIT(26)) | \ + (BIT(27)) | (BIT(30)) | (BIT(31)) | \ (0xffffefffUL << 32)) #ifdef CONFIG_CPU_BIG_ENDIAN @@ -524,23 +524,23 @@ #endif /* SCTLR_EL1 specific flags. */ -#define SCTLR_EL1_UCI (_BITUL(26)) -#define SCTLR_EL1_E0E (_BITUL(24)) -#define SCTLR_EL1_SPAN (_BITUL(23)) -#define SCTLR_EL1_NTWE (_BITUL(18)) -#define SCTLR_EL1_NTWI (_BITUL(16)) -#define SCTLR_EL1_UCT (_BITUL(15)) -#define SCTLR_EL1_DZE (_BITUL(14)) -#define SCTLR_EL1_UMA (_BITUL(9)) -#define SCTLR_EL1_SED (_BITUL(8)) -#define SCTLR_EL1_ITD (_BITUL(7)) -#define SCTLR_EL1_CP15BEN (_BITUL(5)) -#define SCTLR_EL1_SA0 (_BITUL(4)) - -#define SCTLR_EL1_RES1 ((_BITUL(11)) | (_BITUL(20)) | (_BITUL(22)) | (_BITUL(28)) | \ - (_BITUL(29))) -#define SCTLR_EL1_RES0 ((_BITUL(6)) | (_BITUL(10)) | (_BITUL(13)) | (_BITUL(17)) | \ - (_BITUL(27)) | (_BITUL(30)) | (_BITUL(31)) | \ +#define SCTLR_EL1_UCI (BIT(26)) +#define SCTLR_EL1_E0E (BIT(24)) +#define SCTLR_EL1_SPAN (BIT(23)) +#define SCTLR_EL1_NTWE (BIT(18)) +#define SCTLR_EL1_NTWI (BIT(16)) +#define SCTLR_EL1_UCT (BIT(15)) +#define SCTLR_EL1_DZE (BIT(14)) +#define SCTLR_EL1_UMA (BIT(9)) +#define SCTLR_EL1_SED (BIT(8)) +#define SCTLR_EL1_ITD (BIT(7)) +#define SCTLR_EL1_CP15BEN (BIT(5)) +#define SCTLR_EL1_SA0 (BIT(4)) + +#define SCTLR_EL1_RES1 ((BIT(11)) | (BIT(20)) | (BIT(22)) | (BIT(28)) | \ + (BIT(29))) +#define SCTLR_EL1_RES0 ((BIT(6)) | (BIT(10)) | (BIT(13)) | (BIT(17)) | \ + (BIT(27)) | (BIT(30)) | (BIT(31)) | \ (0xffffefffUL << 32)) #ifdef CONFIG_CPU_BIG_ENDIAN @@ -756,13 +756,13 @@ #define ZCR_ELx_LEN_SIZE 9 #define ZCR_ELx_LEN_MASK 0x1ff -#define CPACR_EL1_ZEN_EL1EN (_BITUL(16)) /* enable EL1 access */ -#define CPACR_EL1_ZEN_EL0EN (_BITUL(17)) /* enable EL0 access, if EL1EN set */ +#define CPACR_EL1_ZEN_EL1EN (BIT(16)) /* enable EL1 access */ +#define CPACR_EL1_ZEN_EL0EN (BIT(17)) /* enable EL0 access, if EL1EN set */ #define CPACR_EL1_ZEN (CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN) /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ -#define SYS_MPIDR_SAFE_VAL (_BITUL(31)) +#define SYS_MPIDR_SAFE_VAL (BIT(31)) #ifdef __ASSEMBLY__ diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index c8c61b1eb479..9568c116ac7f 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -59,28 +59,6 @@ static inline const struct fault_info *esr_to_debug_fault_info(unsigned int esr) return debug_fault_info + DBG_ESR_EVT(esr); } -#ifdef CONFIG_KPROBES -static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) -{ - int ret = 0; - - /* kprobe_running() needs smp_processor_id() */ - if (!user_mode(regs)) { - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, esr)) - ret = 1; - preempt_enable(); - } - - return ret; -} -#else -static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) -{ - return 0; -} -#endif - static void data_abort_decode(unsigned int esr) { pr_alert("Data abort info:\n"); @@ -434,7 +412,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, unsigned long vm_flags = VM_READ | VM_WRITE; unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; - if (notify_page_fault(regs, esr)) + if (kprobe_page_fault(regs, esr)) return 0; /* diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 1b49c08dfa2b..e661469cabdd 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -942,6 +942,11 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys) return dt_virt; } +int __init arch_ioremap_p4d_supported(void) +{ + return 0; +} + int __init arch_ioremap_pud_supported(void) { /* diff --git a/arch/h8300/include/asm/bitops.h b/arch/h8300/include/asm/bitops.h index 647a83bd40b7..7aa16c732aa9 100644 --- a/arch/h8300/include/asm/bitops.h +++ b/arch/h8300/include/asm/bitops.h @@ -51,12 +51,6 @@ static inline void FNAME(int nr, volatile unsigned long *addr) \ } \ } -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - H8300_GEN_BITOP(set_bit, "bset") H8300_GEN_BITOP(clear_bit, "bclr") H8300_GEN_BITOP(change_bit, "bnot") diff --git a/arch/hexagon/include/asm/syscall.h b/arch/hexagon/include/asm/syscall.h index 4f054b1ddef5..f6e454f18038 100644 --- a/arch/hexagon/include/asm/syscall.h +++ b/arch/hexagon/include/asm/syscall.h @@ -9,6 +9,8 @@ #define _ASM_HEXAGON_SYSCALL_H #include <uapi/linux/audit.h> +#include <linux/err.h> +#include <asm/ptrace.h> typedef long (*syscall_fn)(unsigned long, unsigned long, unsigned long, unsigned long, @@ -31,6 +33,18 @@ static inline void syscall_get_arguments(struct task_struct *task, memcpy(args, &(®s->r00)[0], 6 * sizeof(args[0])); } +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + return IS_ERR_VALUE(regs->r00) ? regs->r00 : 0; +} + +static inline long syscall_get_return_value(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->r00; +} + static inline int syscall_get_arch(struct task_struct *task) { return AUDIT_ARCH_HEXAGON; diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 3c3a283d3172..c2f299fe9e04 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -21,28 +21,6 @@ extern int die(char *, struct pt_regs *, long); -#ifdef CONFIG_KPROBES -static inline int notify_page_fault(struct pt_regs *regs, int trap) -{ - int ret = 0; - - if (!user_mode(regs)) { - /* kprobe_running() needs smp_processor_id() */ - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, trap)) - ret = 1; - preempt_enable(); - } - - return ret; -} -#else -static inline int notify_page_fault(struct pt_regs *regs, int trap) -{ - return 0; -} -#endif - /* * Return TRUE if ADDRESS points at a page in the kernel's mapped segment * (inside region 5, on ia64) and that page is present. @@ -116,7 +94,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re /* * This is to handle the kprobes on user space access instructions */ - if (notify_page_fault(regs, TRAP_BRKPT)) + if (kprobe_page_fault(regs, TRAP_BRKPT)) return; if (user_mode(regs)) diff --git a/arch/mips/ar7/setup.c b/arch/mips/ar7/setup.c index ba32825ae58a..b3ffe7c898eb 100644 --- a/arch/mips/ar7/setup.c +++ b/arch/mips/ar7/setup.c @@ -57,6 +57,7 @@ const char *get_system_type(void) case TITAN_CHIP_1060: return "TI AR7 (TNETV1060)"; } + /* fall through */ default: return "TI AR7 (unknown)"; } diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index f22538cae0ab..ea385a865781 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -153,7 +153,7 @@ static void __init ath79_detect_sys_type(void) case REV_ID_MAJOR_QCA9533_V2: ver = 2; ath79_soc_rev = 2; - /* drop through */ + /* fall through */ case REV_ID_MAJOR_QCA9533: ath79_soc = ATH79_SOC_QCA9533; diff --git a/arch/mips/bcm63xx/dev-flash.c b/arch/mips/bcm63xx/dev-flash.c index 172dd8397178..a1093934c616 100644 --- a/arch/mips/bcm63xx/dev-flash.c +++ b/arch/mips/bcm63xx/dev-flash.c @@ -94,6 +94,7 @@ static int __init bcm63xx_detect_flash_type(void) case STRAPBUS_6368_BOOT_SEL_PARALLEL: return BCM63XX_FLASH_TYPE_PARALLEL; } + /* fall through */ default: return -EINVAL; } diff --git a/arch/mips/boot/dts/ralink/mt7628a.dtsi b/arch/mips/boot/dts/ralink/mt7628a.dtsi index 9ff7e8faaecc..61f8621e88b3 100644 --- a/arch/mips/boot/dts/ralink/mt7628a.dtsi +++ b/arch/mips/boot/dts/ralink/mt7628a.dtsi @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + / { #address-cells = <1>; #size-cells = <1>; @@ -36,7 +38,113 @@ sysc: system-controller@0 { compatible = "ralink,mt7620a-sysc", "syscon"; - reg = <0x0 0x100>; + reg = <0x0 0x60>; + }; + + pinmux: pinmux@60 { + compatible = "pinctrl-single"; + reg = <0x60 0x8>; + #address-cells = <1>; + #size-cells = <0>; + #pinctrl-cells = <2>; + pinctrl-single,bit-per-mux; + pinctrl-single,register-width = <32>; + pinctrl-single,function-mask = <0x1>; + + pinmux_gpio_gpio: pinmux_gpio_gpio { + pinctrl-single,bits = <0x0 0x0 0x3>; + }; + + pinmux_spi_cs1_cs: pinmux_spi_cs1_cs { + pinctrl-single,bits = <0x0 0x0 0x30>; + }; + + pinmux_i2s_gpio: pinmux_i2s_gpio { + pinctrl-single,bits = <0x0 0x40 0xc0>; + }; + + pinmux_uart0_uart: pinmux_uart0_uart0 { + pinctrl-single,bits = <0x0 0x0 0x300>; + }; + + pinmux_sdmode_sdxc: pinmux_sdmode_sdxc { + pinctrl-single,bits = <0x0 0x0 0xc00>; + }; + + pinmux_sdmode_gpio: pinmux_sdmode_gpio { + pinctrl-single,bits = <0x0 0x400 0xc00>; + }; + + pinmux_spi_spi: pinmux_spi_spi { + pinctrl-single,bits = <0x0 0x0 0x1000>; + }; + + pinmux_refclk_gpio: pinmux_refclk_gpio { + pinctrl-single,bits = <0x0 0x40000 0x40000>; + }; + + pinmux_i2c_i2c: pinmux_i2c_i2c { + pinctrl-single,bits = <0x0 0x0 0x300000>; + }; + + pinmux_uart1_uart: pinmux_uart1_uart1 { + pinctrl-single,bits = <0x0 0x0 0x3000000>; + }; + + pinmux_uart2_uart: pinmux_uart2_uart { + pinctrl-single,bits = <0x0 0x0 0xc000000>; + }; + + pinmux_pwm0_pwm: pinmux_pwm0_pwm { + pinctrl-single,bits = <0x0 0x0 0x30000000>; + }; + + pinmux_pwm0_gpio: pinmux_pwm0_gpio { + pinctrl-single,bits = <0x0 0x10000000 + 0x30000000>; + }; + + pinmux_pwm1_pwm: pinmux_pwm1_pwm { + pinctrl-single,bits = <0x0 0x0 0xc0000000>; + }; + + pinmux_pwm1_gpio: pinmux_pwm1_gpio { + pinctrl-single,bits = <0x0 0x40000000 + 0xc0000000>; + }; + + pinmux_p0led_an_gpio: pinmux_p0led_an_gpio { + pinctrl-single,bits = <0x4 0x4 0xc>; + }; + + pinmux_p1led_an_gpio: pinmux_p1led_an_gpio { + pinctrl-single,bits = <0x4 0x10 0x30>; + }; + + pinmux_p2led_an_gpio: pinmux_p2led_an_gpio { + pinctrl-single,bits = <0x4 0x40 0xc0>; + }; + + pinmux_p3led_an_gpio: pinmux_p3led_an_gpio { + pinctrl-single,bits = <0x4 0x100 0x300>; + }; + + pinmux_p4led_an_gpio: pinmux_p4led_an_gpio { + pinctrl-single,bits = <0x4 0x400 0xc00>; + }; + }; + + watchdog: watchdog@100 { + compatible = "mediatek,mt7621-wdt"; + reg = <0x100 0x30>; + + resets = <&resetc 8>; + reset-names = "wdt"; + + interrupt-parent = <&intc>; + interrupts = <24>; + + status = "disabled"; }; intc: interrupt-controller@200 { @@ -62,10 +170,42 @@ reg = <0x300 0x100>; }; + gpio: gpio@600 { + compatible = "mediatek,mt7621-gpio"; + reg = <0x600 0x100>; + + gpio-controller; + interrupt-controller; + #gpio-cells = <2>; + #interrupt-cells = <2>; + + interrupt-parent = <&intc>; + interrupts = <6>; + }; + + spi: spi@b00 { + compatible = "ralink,mt7621-spi"; + reg = <0xb00 0x100>; + + pinctrl-names = "default"; + pinctrl-0 = <&pinmux_spi_spi>; + + resets = <&resetc 18>; + reset-names = "spi"; + + #address-cells = <1>; + #size-cells = <0>; + + status = "disabled"; + }; + uart0: uartlite@c00 { compatible = "ns16550a"; reg = <0xc00 0x100>; + pinctrl-names = "default"; + pinctrl-0 = <&pinmux_uart0_uart>; + resets = <&resetc 12>; reset-names = "uart0"; @@ -79,6 +219,9 @@ compatible = "ns16550a"; reg = <0xd00 0x100>; + pinctrl-names = "default"; + pinctrl-0 = <&pinmux_uart1_uart>; + resets = <&resetc 19>; reset-names = "uart1"; @@ -92,6 +235,9 @@ compatible = "ns16550a"; reg = <0xe00 0x100>; + pinctrl-names = "default"; + pinctrl-0 = <&pinmux_uart2_uart>; + resets = <&resetc 20>; reset-names = "uart2"; diff --git a/arch/mips/cavium-octeon/executive/cvmx-pko.c b/arch/mips/cavium-octeon/executive/cvmx-pko.c index 676fab50dd2b..b077597c668a 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-pko.c +++ b/arch/mips/cavium-octeon/executive/cvmx-pko.c @@ -485,11 +485,11 @@ cvmx_pko_status_t cvmx_pko_config_port(uint64_t port, uint64_t base_queue, config.s.qos_mask = 0xff; break; case CVMX_PKO_QUEUE_STATIC_PRIORITY: - /* Pass 1 will fall through to the error case */ if (!cvmx_octeon_is_pass1()) { config.s.qos_mask = 0xff; break; } + /* fall through - to the error case, when Pass 1 */ default: cvmx_dprintf("ERROR: cvmx_pko_config_port: Invalid " "priority %llu\n", diff --git a/arch/mips/configs/ar7_defconfig b/arch/mips/configs/ar7_defconfig index c83fdf649327..cef2754bd408 100644 --- a/arch/mips/configs/ar7_defconfig +++ b/arch/mips/configs/ar7_defconfig @@ -71,7 +71,6 @@ CONFIG_NET_ACT_POLICE=y CONFIG_HAMRADIO=y CONFIG_CFG80211=m CONFIG_MAC80211=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y diff --git a/arch/mips/configs/ath25_defconfig b/arch/mips/configs/ath25_defconfig index 5dd6b1939e9c..c35add2fd716 100644 --- a/arch/mips/configs/ath25_defconfig +++ b/arch/mips/configs/ath25_defconfig @@ -37,7 +37,6 @@ CONFIG_IP_ADVANCED_ROUTER=y CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_MAC80211_DEBUGFS=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_REDBOOT_PARTS=y CONFIG_MTD_REDBOOT_DIRECTORY_BLOCK=-2 diff --git a/arch/mips/configs/ath79_defconfig b/arch/mips/configs/ath79_defconfig index 6f981af67826..4ffc59cab436 100644 --- a/arch/mips/configs/ath79_defconfig +++ b/arch/mips/configs/ath79_defconfig @@ -37,7 +37,6 @@ CONFIG_IP_ADVANCED_ROUTER=y CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_MAC80211_DEBUGFS=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_REDBOOT_PARTS=y CONFIG_MTD_REDBOOT_DIRECTORY_BLOCK=-2 diff --git a/arch/mips/configs/bcm63xx_defconfig b/arch/mips/configs/bcm63xx_defconfig index d22fe62adad3..54e2f9a659fb 100644 --- a/arch/mips/configs/bcm63xx_defconfig +++ b/arch/mips/configs/bcm63xx_defconfig @@ -34,7 +34,6 @@ CONFIG_INET=y CONFIG_CFG80211=y CONFIG_NL80211_TESTMODE=y CONFIG_MAC80211=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set CONFIG_MTD=y diff --git a/arch/mips/configs/bigsur_defconfig b/arch/mips/configs/bigsur_defconfig index 597bc0aa2653..66566026409d 100644 --- a/arch/mips/configs/bigsur_defconfig +++ b/arch/mips/configs/bigsur_defconfig @@ -99,7 +99,6 @@ CONFIG_BPQETHER=m CONFIG_BAYCOM_SER_FDX=m CONFIG_BAYCOM_SER_HDX=m CONFIG_YAM=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_CRYPTOLOOP=m diff --git a/arch/mips/configs/bmips_be_defconfig b/arch/mips/configs/bmips_be_defconfig index 8a91f0101134..f669a40e085b 100644 --- a/arch/mips/configs/bmips_be_defconfig +++ b/arch/mips/configs/bmips_be_defconfig @@ -26,7 +26,6 @@ CONFIG_INET=y CONFIG_CFG80211=y CONFIG_NL80211_TESTMODE=y CONFIG_MAC80211=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set diff --git a/arch/mips/configs/bmips_stb_defconfig b/arch/mips/configs/bmips_stb_defconfig index 39adcca46bb0..a0b775893dba 100644 --- a/arch/mips/configs/bmips_stb_defconfig +++ b/arch/mips/configs/bmips_stb_defconfig @@ -35,7 +35,6 @@ CONFIG_INET=y CONFIG_CFG80211=y CONFIG_NL80211_TESTMODE=y CONFIG_MAC80211=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/cavium_octeon_defconfig index d7abb648b8a0..b6695367aa33 100644 --- a/arch/mips/configs/cavium_octeon_defconfig +++ b/arch/mips/configs/cavium_octeon_defconfig @@ -42,7 +42,6 @@ CONFIG_IP_MROUTE=y CONFIG_IP_PIMSM_V1=y CONFIG_IP_PIMSM_V2=y CONFIG_SYN_COOKIES=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y # CONFIG_FW_LOADER is not set CONFIG_MTD=y diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig index 50bebce28500..cb4aa23a2bf4 100644 --- a/arch/mips/configs/ci20_defconfig +++ b/arch/mips/configs/ci20_defconfig @@ -44,7 +44,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y # CONFIG_FW_LOADER is not set # CONFIG_ALLOW_DEV_COREDUMP is not set diff --git a/arch/mips/configs/cobalt_defconfig b/arch/mips/configs/cobalt_defconfig index 20c62841827f..c6a652ad34f7 100644 --- a/arch/mips/configs/cobalt_defconfig +++ b/arch/mips/configs/cobalt_defconfig @@ -14,7 +14,6 @@ CONFIG_NET_KEY=y CONFIG_NET_KEY_MIGRATE=y CONFIG_INET=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_JEDECPROBE=y CONFIG_MTD_CFI_AMDSTD=y diff --git a/arch/mips/configs/fuloong2e_defconfig b/arch/mips/configs/fuloong2e_defconfig index 8bcb61a6ec15..7a7af706e898 100644 --- a/arch/mips/configs/fuloong2e_defconfig +++ b/arch/mips/configs/fuloong2e_defconfig @@ -83,7 +83,6 @@ CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_PHONET=m CONFIG_NET_9P=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_MTD=m CONFIG_MTD_BLOCK=m diff --git a/arch/mips/configs/gpr_defconfig b/arch/mips/configs/gpr_defconfig index 9d9af5f923c3..9085f4d6c698 100644 --- a/arch/mips/configs/gpr_defconfig +++ b/arch/mips/configs/gpr_defconfig @@ -249,7 +249,6 @@ CONFIG_WATCHDOG_NOWAYOUT=y CONFIG_SSB=m CONFIG_SSB_DRIVER_PCICORE=y # CONFIG_VGA_ARB is not set -CONFIG_BACKLIGHT_LCD_SUPPORT=y # CONFIG_LCD_CLASS_DEVICE is not set CONFIG_BACKLIGHT_CLASS_DEVICE=y # CONFIG_BACKLIGHT_GENERIC is not set diff --git a/arch/mips/configs/ip27_defconfig b/arch/mips/configs/ip27_defconfig index 54db5dedf776..82d942a6026e 100644 --- a/arch/mips/configs/ip27_defconfig +++ b/arch/mips/configs/ip27_defconfig @@ -91,7 +91,6 @@ CONFIG_NET_ACT_SKBEDIT=m CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_RFKILL=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_CDROM_PKTCDVD=m diff --git a/arch/mips/configs/ip32_defconfig b/arch/mips/configs/ip32_defconfig index 8f6d8af2e3c0..572cab91670c 100644 --- a/arch/mips/configs/ip32_defconfig +++ b/arch/mips/configs/ip32_defconfig @@ -42,7 +42,6 @@ CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_TUNNEL=m CONFIG_NETWORK_SECMARK=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_CONNECTOR=y CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_CRYPTOLOOP=m diff --git a/arch/mips/configs/lemote2f_defconfig b/arch/mips/configs/lemote2f_defconfig index 300127b0f5b7..d44f1469cf64 100644 --- a/arch/mips/configs/lemote2f_defconfig +++ b/arch/mips/configs/lemote2f_defconfig @@ -77,7 +77,6 @@ CONFIG_MAC80211=m CONFIG_MAC80211_LEDS=y CONFIG_RFKILL=m CONFIG_RFKILL_INPUT=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_RAM=y @@ -144,7 +143,6 @@ CONFIG_FB_TILEBLITTING=y CONFIG_FB_SIS=y CONFIG_FB_SIS_300=y CONFIG_FB_SIS_315=y -CONFIG_BACKLIGHT_LCD_SUPPORT=y # CONFIG_LCD_CLASS_DEVICE is not set CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_BACKLIGHT_GENERIC=m diff --git a/arch/mips/configs/loongson1b_defconfig b/arch/mips/configs/loongson1b_defconfig index 3d390a7494d6..25e70423e17d 100644 --- a/arch/mips/configs/loongson1b_defconfig +++ b/arch/mips/configs/loongson1b_defconfig @@ -34,7 +34,6 @@ CONFIG_SYN_COOKIES=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set diff --git a/arch/mips/configs/loongson1c_defconfig b/arch/mips/configs/loongson1c_defconfig index 247d56e94c0a..3a158d4d2fab 100644 --- a/arch/mips/configs/loongson1c_defconfig +++ b/arch/mips/configs/loongson1c_defconfig @@ -35,7 +35,6 @@ CONFIG_SYN_COOKIES=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig index 1322adb705c8..90ee0084d786 100644 --- a/arch/mips/configs/loongson3_defconfig +++ b/arch/mips/configs/loongson3_defconfig @@ -97,7 +97,6 @@ CONFIG_CFG80211_WEXT=y CONFIG_MAC80211=m CONFIG_RFKILL=m CONFIG_RFKILL_INPUT=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_MTD=m diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig index 0de92ac1ca64..59eedf55419d 100644 --- a/arch/mips/configs/malta_defconfig +++ b/arch/mips/configs/malta_defconfig @@ -214,7 +214,6 @@ CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_MAC80211_MESH=y CONFIG_RFKILL=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_CONNECTOR=m CONFIG_MTD=y diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig index efc3abace048..8ef612552a19 100644 --- a/arch/mips/configs/malta_kvm_defconfig +++ b/arch/mips/configs/malta_kvm_defconfig @@ -219,7 +219,6 @@ CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_MAC80211_MESH=y CONFIG_RFKILL=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_CONNECTOR=m CONFIG_MTD=y diff --git a/arch/mips/configs/malta_kvm_guest_defconfig b/arch/mips/configs/malta_kvm_guest_defconfig index c6ceeca4394d..d2a008c9907c 100644 --- a/arch/mips/configs/malta_kvm_guest_defconfig +++ b/arch/mips/configs/malta_kvm_guest_defconfig @@ -216,7 +216,6 @@ CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_MAC80211_MESH=y CONFIG_RFKILL=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_CONNECTOR=m CONFIG_MTD=y diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig index 56861aef2756..970df6d42728 100644 --- a/arch/mips/configs/maltaup_xpa_defconfig +++ b/arch/mips/configs/maltaup_xpa_defconfig @@ -216,7 +216,6 @@ CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_MAC80211_MESH=y CONFIG_RFKILL=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_CONNECTOR=m diff --git a/arch/mips/configs/mips_paravirt_defconfig b/arch/mips/configs/mips_paravirt_defconfig index 8dc5d96a08de..5599cde97030 100644 --- a/arch/mips/configs/mips_paravirt_defconfig +++ b/arch/mips/configs/mips_paravirt_defconfig @@ -39,7 +39,6 @@ CONFIG_IP_PIMSM_V1=y CONFIG_IP_PIMSM_V2=y CONFIG_SYN_COOKIES=y # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_LOOP=y CONFIG_VIRTIO_BLK=y diff --git a/arch/mips/configs/omega2p_defconfig b/arch/mips/configs/omega2p_defconfig index 0649b8f06b7c..a39426e57e91 100644 --- a/arch/mips/configs/omega2p_defconfig +++ b/arch/mips/configs/omega2p_defconfig @@ -42,7 +42,6 @@ CONFIG_INET=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y # CONFIG_FW_LOADER is not set # CONFIG_ALLOW_DEV_COREDUMP is not set diff --git a/arch/mips/configs/pistachio_defconfig b/arch/mips/configs/pistachio_defconfig index 2f08d071ada6..24e07180c57d 100644 --- a/arch/mips/configs/pistachio_defconfig +++ b/arch/mips/configs/pistachio_defconfig @@ -214,7 +214,6 @@ CONFIG_IR_IMG_RC6=y CONFIG_MEDIA_SUPPORT=y CONFIG_FB=y CONFIG_FB_MODE_HELPERS=y -CONFIG_BACKLIGHT_LCD_SUPPORT=y # CONFIG_LCD_CLASS_DEVICE is not set CONFIG_BACKLIGHT_CLASS_DEVICE=y CONFIG_SOUND=y diff --git a/arch/mips/configs/pnx8335_stb225_defconfig b/arch/mips/configs/pnx8335_stb225_defconfig index aa0b169800e0..738ba3b1374b 100644 --- a/arch/mips/configs/pnx8335_stb225_defconfig +++ b/arch/mips/configs/pnx8335_stb225_defconfig @@ -25,7 +25,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_INET_AH=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/mips/configs/qi_lb60_defconfig b/arch/mips/configs/qi_lb60_defconfig index 1a0677d04982..208da8a55f48 100644 --- a/arch/mips/configs/qi_lb60_defconfig +++ b/arch/mips/configs/qi_lb60_defconfig @@ -41,7 +41,6 @@ CONFIG_TCP_CONG_ADVANCED=y CONFIG_TCP_CONG_WESTWOOD=y # CONFIG_TCP_CONG_HTCP is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_RAW_NAND=y @@ -77,7 +76,6 @@ CONFIG_REGULATOR=y CONFIG_REGULATOR_FIXED_VOLTAGE=y CONFIG_FB=y CONFIG_FB_JZ4740=y -CONFIG_BACKLIGHT_LCD_SUPPORT=y CONFIG_LCD_CLASS_DEVICE=y # CONFIG_BACKLIGHT_CLASS_DEVICE is not set # CONFIG_VGA_CONSOLE is not set diff --git a/arch/mips/configs/rb532_defconfig b/arch/mips/configs/rb532_defconfig index 864c70fbe668..5b947183852b 100644 --- a/arch/mips/configs/rb532_defconfig +++ b/arch/mips/configs/rb532_defconfig @@ -104,7 +104,6 @@ CONFIG_NET_ACT_MIRRED=m CONFIG_NET_ACT_IPT=m CONFIG_NET_ACT_PEDIT=m CONFIG_HAMRADIO=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_BLOCK2MTD=y diff --git a/arch/mips/configs/rt305x_defconfig b/arch/mips/configs/rt305x_defconfig index 0392e38010e6..110948bc6b39 100644 --- a/arch/mips/configs/rt305x_defconfig +++ b/arch/mips/configs/rt305x_defconfig @@ -69,7 +69,6 @@ CONFIG_BRIDGE=y CONFIG_VLAN_8021Q=y CONFIG_NET_SCHED=y CONFIG_HAMRADIO=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/mips/configs/sb1250_swarm_defconfig b/arch/mips/configs/sb1250_swarm_defconfig index ad8981666ee4..6883ea4477d4 100644 --- a/arch/mips/configs/sb1250_swarm_defconfig +++ b/arch/mips/configs/sb1250_swarm_defconfig @@ -43,7 +43,6 @@ CONFIG_NETWORK_SECMARK=y CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_RFKILL=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_CONNECTOR=m CONFIG_BLK_DEV_RAM=y diff --git a/arch/mips/configs/tb0219_defconfig b/arch/mips/configs/tb0219_defconfig index f0a11a72307e..6547f84750b5 100644 --- a/arch/mips/configs/tb0219_defconfig +++ b/arch/mips/configs/tb0219_defconfig @@ -28,7 +28,6 @@ CONFIG_SYN_COOKIES=y # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_NETWORK_SECMARK=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y diff --git a/arch/mips/configs/tb0226_defconfig b/arch/mips/configs/tb0226_defconfig index 025e45656359..7e099f7c2286 100644 --- a/arch/mips/configs/tb0226_defconfig +++ b/arch/mips/configs/tb0226_defconfig @@ -26,7 +26,6 @@ CONFIG_SYN_COOKIES=y # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_NETWORK_SECMARK=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y diff --git a/arch/mips/configs/tb0287_defconfig b/arch/mips/configs/tb0287_defconfig index 68490248e3f1..0d881dd862c0 100644 --- a/arch/mips/configs/tb0287_defconfig +++ b/arch/mips/configs/tb0287_defconfig @@ -30,7 +30,6 @@ CONFIG_TCP_CONG_BIC=y CONFIG_TCP_CONG_CUBIC=m # CONFIG_IPV6 is not set CONFIG_NETWORK_SECMARK=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y diff --git a/arch/mips/configs/vocore2_defconfig b/arch/mips/configs/vocore2_defconfig index ded3dce911d5..523b944fd527 100644 --- a/arch/mips/configs/vocore2_defconfig +++ b/arch/mips/configs/vocore2_defconfig @@ -42,7 +42,6 @@ CONFIG_INET=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y # CONFIG_FW_LOADER is not set # CONFIG_ALLOW_DEV_COREDUMP is not set diff --git a/arch/mips/configs/xway_defconfig b/arch/mips/configs/xway_defconfig index 203db83c3ee9..49b5ea6eff62 100644 --- a/arch/mips/configs/xway_defconfig +++ b/arch/mips/configs/xway_defconfig @@ -71,7 +71,6 @@ CONFIG_BRIDGE=y CONFIG_VLAN_8021Q=y CONFIG_NET_SCHED=y CONFIG_HAMRADIO=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h index 6ad7d3cabd91..290369fa44a4 100644 --- a/arch/mips/include/asm/cpu.h +++ b/arch/mips/include/asm/cpu.h @@ -9,6 +9,8 @@ #ifndef _ASM_CPU_H #define _ASM_CPU_H +#include <linux/bits.h> + /* As of the MIPS32 and MIPS64 specs from MTI, the PRId register (CP0 register 15, select 0) is defined in this (backwards compatible) way: @@ -353,76 +355,69 @@ enum cpu_type_enum { MIPS_CPU_ISA_M64R6) /* - * Private version of BIT_ULL() to escape include file recursion hell. - * We soon will have to switch to another mechanism that will work with - * more than 64 bits anyway. - */ -#define MBIT_ULL(bit) (1ULL << (bit)) - -/* * CPU Option encodings */ -#define MIPS_CPU_TLB MBIT_ULL( 0) /* CPU has TLB */ -#define MIPS_CPU_4KEX MBIT_ULL( 1) /* "R4K" exception model */ -#define MIPS_CPU_3K_CACHE MBIT_ULL( 2) /* R3000-style caches */ -#define MIPS_CPU_4K_CACHE MBIT_ULL( 3) /* R4000-style caches */ -#define MIPS_CPU_TX39_CACHE MBIT_ULL( 4) /* TX3900-style caches */ -#define MIPS_CPU_FPU MBIT_ULL( 5) /* CPU has FPU */ -#define MIPS_CPU_32FPR MBIT_ULL( 6) /* 32 dbl. prec. FP registers */ -#define MIPS_CPU_COUNTER MBIT_ULL( 7) /* Cycle count/compare */ -#define MIPS_CPU_WATCH MBIT_ULL( 8) /* watchpoint registers */ -#define MIPS_CPU_DIVEC MBIT_ULL( 9) /* dedicated interrupt vector */ -#define MIPS_CPU_VCE MBIT_ULL(10) /* virt. coherence conflict possible */ -#define MIPS_CPU_CACHE_CDEX_P MBIT_ULL(11) /* Create_Dirty_Exclusive CACHE op */ -#define MIPS_CPU_CACHE_CDEX_S MBIT_ULL(12) /* ... same for seconary cache ... */ -#define MIPS_CPU_MCHECK MBIT_ULL(13) /* Machine check exception */ -#define MIPS_CPU_EJTAG MBIT_ULL(14) /* EJTAG exception */ -#define MIPS_CPU_NOFPUEX MBIT_ULL(15) /* no FPU exception */ -#define MIPS_CPU_LLSC MBIT_ULL(16) /* CPU has ll/sc instructions */ -#define MIPS_CPU_INCLUSIVE_CACHES MBIT_ULL(17) /* P-cache subset enforced */ -#define MIPS_CPU_PREFETCH MBIT_ULL(18) /* CPU has usable prefetch */ -#define MIPS_CPU_VINT MBIT_ULL(19) /* CPU supports MIPSR2 vectored interrupts */ -#define MIPS_CPU_VEIC MBIT_ULL(20) /* CPU supports MIPSR2 external interrupt controller mode */ -#define MIPS_CPU_ULRI MBIT_ULL(21) /* CPU has ULRI feature */ -#define MIPS_CPU_PCI MBIT_ULL(22) /* CPU has Perf Ctr Int indicator */ -#define MIPS_CPU_RIXI MBIT_ULL(23) /* CPU has TLB Read/eXec Inhibit */ -#define MIPS_CPU_MICROMIPS MBIT_ULL(24) /* CPU has microMIPS capability */ -#define MIPS_CPU_TLBINV MBIT_ULL(25) /* CPU supports TLBINV/F */ -#define MIPS_CPU_SEGMENTS MBIT_ULL(26) /* CPU supports Segmentation Control registers */ -#define MIPS_CPU_EVA MBIT_ULL(27) /* CPU supports Enhanced Virtual Addressing */ -#define MIPS_CPU_HTW MBIT_ULL(28) /* CPU support Hardware Page Table Walker */ -#define MIPS_CPU_RIXIEX MBIT_ULL(29) /* CPU has unique exception codes for {Read, Execute}-Inhibit exceptions */ -#define MIPS_CPU_MAAR MBIT_ULL(30) /* MAAR(I) registers are present */ -#define MIPS_CPU_FRE MBIT_ULL(31) /* FRE & UFE bits implemented */ -#define MIPS_CPU_RW_LLB MBIT_ULL(32) /* LLADDR/LLB writes are allowed */ -#define MIPS_CPU_LPA MBIT_ULL(33) /* CPU supports Large Physical Addressing */ -#define MIPS_CPU_CDMM MBIT_ULL(34) /* CPU has Common Device Memory Map */ -#define MIPS_CPU_BP_GHIST MBIT_ULL(35) /* R12K+ Branch Prediction Global History */ -#define MIPS_CPU_SP MBIT_ULL(36) /* Small (1KB) page support */ -#define MIPS_CPU_FTLB MBIT_ULL(37) /* CPU has Fixed-page-size TLB */ -#define MIPS_CPU_NAN_LEGACY MBIT_ULL(38) /* Legacy NaN implemented */ -#define MIPS_CPU_NAN_2008 MBIT_ULL(39) /* 2008 NaN implemented */ -#define MIPS_CPU_VP MBIT_ULL(40) /* MIPSr6 Virtual Processors (multi-threading) */ -#define MIPS_CPU_LDPTE MBIT_ULL(41) /* CPU has ldpte/lddir instructions */ -#define MIPS_CPU_MVH MBIT_ULL(42) /* CPU supports MFHC0/MTHC0 */ -#define MIPS_CPU_EBASE_WG MBIT_ULL(43) /* CPU has EBase.WG */ -#define MIPS_CPU_BADINSTR MBIT_ULL(44) /* CPU has BadInstr register */ -#define MIPS_CPU_BADINSTRP MBIT_ULL(45) /* CPU has BadInstrP register */ -#define MIPS_CPU_CTXTC MBIT_ULL(46) /* CPU has [X]ConfigContext registers */ -#define MIPS_CPU_PERF MBIT_ULL(47) /* CPU has MIPS performance counters */ -#define MIPS_CPU_GUESTCTL0EXT MBIT_ULL(48) /* CPU has VZ GuestCtl0Ext register */ -#define MIPS_CPU_GUESTCTL1 MBIT_ULL(49) /* CPU has VZ GuestCtl1 register */ -#define MIPS_CPU_GUESTCTL2 MBIT_ULL(50) /* CPU has VZ GuestCtl2 register */ -#define MIPS_CPU_GUESTID MBIT_ULL(51) /* CPU uses VZ ASE GuestID feature */ -#define MIPS_CPU_DRG MBIT_ULL(52) /* CPU has VZ Direct Root to Guest (DRG) */ -#define MIPS_CPU_UFR MBIT_ULL(53) /* CPU supports User mode FR switching */ +#define MIPS_CPU_TLB BIT_ULL( 0) /* CPU has TLB */ +#define MIPS_CPU_4KEX BIT_ULL( 1) /* "R4K" exception model */ +#define MIPS_CPU_3K_CACHE BIT_ULL( 2) /* R3000-style caches */ +#define MIPS_CPU_4K_CACHE BIT_ULL( 3) /* R4000-style caches */ +#define MIPS_CPU_TX39_CACHE BIT_ULL( 4) /* TX3900-style caches */ +#define MIPS_CPU_FPU BIT_ULL( 5) /* CPU has FPU */ +#define MIPS_CPU_32FPR BIT_ULL( 6) /* 32 dbl. prec. FP registers */ +#define MIPS_CPU_COUNTER BIT_ULL( 7) /* Cycle count/compare */ +#define MIPS_CPU_WATCH BIT_ULL( 8) /* watchpoint registers */ +#define MIPS_CPU_DIVEC BIT_ULL( 9) /* dedicated interrupt vector */ +#define MIPS_CPU_VCE BIT_ULL(10) /* virt. coherence conflict possible */ +#define MIPS_CPU_CACHE_CDEX_P BIT_ULL(11) /* Create_Dirty_Exclusive CACHE op */ +#define MIPS_CPU_CACHE_CDEX_S BIT_ULL(12) /* ... same for seconary cache ... */ +#define MIPS_CPU_MCHECK BIT_ULL(13) /* Machine check exception */ +#define MIPS_CPU_EJTAG BIT_ULL(14) /* EJTAG exception */ +#define MIPS_CPU_NOFPUEX BIT_ULL(15) /* no FPU exception */ +#define MIPS_CPU_LLSC BIT_ULL(16) /* CPU has ll/sc instructions */ +#define MIPS_CPU_INCLUSIVE_CACHES BIT_ULL(17) /* P-cache subset enforced */ +#define MIPS_CPU_PREFETCH BIT_ULL(18) /* CPU has usable prefetch */ +#define MIPS_CPU_VINT BIT_ULL(19) /* CPU supports MIPSR2 vectored interrupts */ +#define MIPS_CPU_VEIC BIT_ULL(20) /* CPU supports MIPSR2 external interrupt controller mode */ +#define MIPS_CPU_ULRI BIT_ULL(21) /* CPU has ULRI feature */ +#define MIPS_CPU_PCI BIT_ULL(22) /* CPU has Perf Ctr Int indicator */ +#define MIPS_CPU_RIXI BIT_ULL(23) /* CPU has TLB Read/eXec Inhibit */ +#define MIPS_CPU_MICROMIPS BIT_ULL(24) /* CPU has microMIPS capability */ +#define MIPS_CPU_TLBINV BIT_ULL(25) /* CPU supports TLBINV/F */ +#define MIPS_CPU_SEGMENTS BIT_ULL(26) /* CPU supports Segmentation Control registers */ +#define MIPS_CPU_EVA BIT_ULL(27) /* CPU supports Enhanced Virtual Addressing */ +#define MIPS_CPU_HTW BIT_ULL(28) /* CPU support Hardware Page Table Walker */ +#define MIPS_CPU_RIXIEX BIT_ULL(29) /* CPU has unique exception codes for {Read, Execute}-Inhibit exceptions */ +#define MIPS_CPU_MAAR BIT_ULL(30) /* MAAR(I) registers are present */ +#define MIPS_CPU_FRE BIT_ULL(31) /* FRE & UFE bits implemented */ +#define MIPS_CPU_RW_LLB BIT_ULL(32) /* LLADDR/LLB writes are allowed */ +#define MIPS_CPU_LPA BIT_ULL(33) /* CPU supports Large Physical Addressing */ +#define MIPS_CPU_CDMM BIT_ULL(34) /* CPU has Common Device Memory Map */ +#define MIPS_CPU_BP_GHIST BIT_ULL(35) /* R12K+ Branch Prediction Global History */ +#define MIPS_CPU_SP BIT_ULL(36) /* Small (1KB) page support */ +#define MIPS_CPU_FTLB BIT_ULL(37) /* CPU has Fixed-page-size TLB */ +#define MIPS_CPU_NAN_LEGACY BIT_ULL(38) /* Legacy NaN implemented */ +#define MIPS_CPU_NAN_2008 BIT_ULL(39) /* 2008 NaN implemented */ +#define MIPS_CPU_VP BIT_ULL(40) /* MIPSr6 Virtual Processors (multi-threading) */ +#define MIPS_CPU_LDPTE BIT_ULL(41) /* CPU has ldpte/lddir instructions */ +#define MIPS_CPU_MVH BIT_ULL(42) /* CPU supports MFHC0/MTHC0 */ +#define MIPS_CPU_EBASE_WG BIT_ULL(43) /* CPU has EBase.WG */ +#define MIPS_CPU_BADINSTR BIT_ULL(44) /* CPU has BadInstr register */ +#define MIPS_CPU_BADINSTRP BIT_ULL(45) /* CPU has BadInstrP register */ +#define MIPS_CPU_CTXTC BIT_ULL(46) /* CPU has [X]ConfigContext registers */ +#define MIPS_CPU_PERF BIT_ULL(47) /* CPU has MIPS performance counters */ +#define MIPS_CPU_GUESTCTL0EXT BIT_ULL(48) /* CPU has VZ GuestCtl0Ext register */ +#define MIPS_CPU_GUESTCTL1 BIT_ULL(49) /* CPU has VZ GuestCtl1 register */ +#define MIPS_CPU_GUESTCTL2 BIT_ULL(50) /* CPU has VZ GuestCtl2 register */ +#define MIPS_CPU_GUESTID BIT_ULL(51) /* CPU uses VZ ASE GuestID feature */ +#define MIPS_CPU_DRG BIT_ULL(52) /* CPU has VZ Direct Root to Guest (DRG) */ +#define MIPS_CPU_UFR BIT_ULL(53) /* CPU supports User mode FR switching */ #define MIPS_CPU_SHARED_FTLB_RAM \ - MBIT_ULL(54) /* CPU shares FTLB RAM with another */ + BIT_ULL(54) /* CPU shares FTLB RAM with another */ #define MIPS_CPU_SHARED_FTLB_ENTRIES \ - MBIT_ULL(55) /* CPU shares FTLB entries with another */ + BIT_ULL(55) /* CPU shares FTLB entries with another */ #define MIPS_CPU_MT_PER_TC_PERF_COUNTERS \ - MBIT_ULL(56) /* CPU has perf counters implemented per TC (MIPSMT ASE) */ -#define MIPS_CPU_MMID MBIT_ULL(57) /* CPU supports MemoryMapIDs */ + BIT_ULL(56) /* CPU has perf counters implemented per TC (MIPSMT ASE) */ +#define MIPS_CPU_MMID BIT_ULL(57) /* CPU supports MemoryMapIDs */ /* * CPU ASE encodings diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index 29997e42480e..97a280640daf 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -149,8 +149,6 @@ static inline void *isa_bus_to_virt(unsigned long address) return phys_to_virt(address); } -#define isa_page_to_bus page_to_phys - /* * However PCI ones are not necessarily 1:1 and therefore these interfaces * are forbidden in portable PCI drivers. @@ -462,7 +460,12 @@ __BUILD_MEMORY_PFX(, bwlq, type, 0) BUILDIO_MEM(b, u8) BUILDIO_MEM(w, u16) BUILDIO_MEM(l, u32) +#ifdef CONFIG_64BIT BUILDIO_MEM(q, u64) +#else +__BUILD_MEMORY_PFX(__raw_, q, u64, 0) +__BUILD_MEMORY_PFX(__mem_, q, u64, 0) +#endif #define __BUILD_IOPORT_PFX(bus, bwlq, type) \ __BUILD_IOPORT_SINGLE(bus, bwlq, type, 1, 0,) \ @@ -488,12 +491,16 @@ __BUILDIO(q, u64) #define readb_relaxed __relaxed_readb #define readw_relaxed __relaxed_readw #define readl_relaxed __relaxed_readl +#ifdef CONFIG_64BIT #define readq_relaxed __relaxed_readq +#endif #define writeb_relaxed __relaxed_writeb #define writew_relaxed __relaxed_writew #define writel_relaxed __relaxed_writel +#ifdef CONFIG_64BIT #define writeq_relaxed __relaxed_writeq +#endif #define readb_be(addr) \ __raw_readb((__force unsigned *)(addr)) @@ -516,8 +523,10 @@ __BUILDIO(q, u64) /* * Some code tests for these symbols */ +#ifdef CONFIG_64BIT #define readq readq #define writeq writeq +#endif #define __BUILD_MEMORY_STRING(bwlq, type) \ \ diff --git a/arch/mips/include/asm/kprobes.h b/arch/mips/include/asm/kprobes.h index 3cf8e4d5fa28..68b1e5d458cf 100644 --- a/arch/mips/include/asm/kprobes.h +++ b/arch/mips/include/asm/kprobes.h @@ -41,6 +41,7 @@ do { \ #define kretprobe_blacklist_size 0 void arch_remove_kprobe(struct kprobe *p); +int kprobe_fault_handler(struct pt_regs *regs, int trapnr); /* Architecture specific copy of original instruction*/ struct arch_specific_insn { diff --git a/arch/mips/include/asm/mach-jz4740/clock.h b/arch/mips/include/asm/mach-jz4740/clock.h deleted file mode 100644 index 600d5051691a..000000000000 --- a/arch/mips/include/asm/mach-jz4740/clock.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2010, Lars-Peter Clausen <lars@metafoo.de> - */ - -#ifndef __ASM_JZ4740_CLOCK_H__ -#define __ASM_JZ4740_CLOCK_H__ - -enum jz4740_wait_mode { - JZ4740_WAIT_MODE_IDLE, - JZ4740_WAIT_MODE_SLEEP, -}; - -void jz4740_clock_set_wait_mode(enum jz4740_wait_mode mode); - -void jz4740_clock_suspend(void); -void jz4740_clock_resume(void); - -void jz4740_clock_udc_enable_auto_suspend(void); -void jz4740_clock_udc_disable_auto_suspend(void); - -#endif diff --git a/arch/mips/include/asm/mach-ralink/pinmux.h b/arch/mips/include/asm/mach-ralink/pinmux.h index e54d4e1533b4..048309348be0 100644 --- a/arch/mips/include/asm/mach-ralink/pinmux.h +++ b/arch/mips/include/asm/mach-ralink/pinmux.h @@ -1,6 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * * Copyright (C) 2012 John Crispin <john@phrozen.org> */ diff --git a/arch/mips/include/asm/syscall.h b/arch/mips/include/asm/syscall.h index acf80ae0a430..83bb439597d8 100644 --- a/arch/mips/include/asm/syscall.h +++ b/arch/mips/include/asm/syscall.h @@ -89,6 +89,12 @@ static inline unsigned long mips_get_syscall_arg(unsigned long *arg, unreachable(); } +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->regs[7] ? -regs->regs[2] : 0; +} + static inline long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { diff --git a/arch/mips/jz4740/board-qi_lb60.c b/arch/mips/jz4740/board-qi_lb60.c index 071e9d94eea7..4a7a80c358c7 100644 --- a/arch/mips/jz4740/board-qi_lb60.c +++ b/arch/mips/jz4740/board-qi_lb60.c @@ -37,8 +37,6 @@ #include <asm/mach-jz4740/platform.h> -#include "clock.h" - /* GPIOs */ #define QI_LB60_GPIO_KEYOUT(x) (JZ_GPIO_PORTC(10) + (x)) #define QI_LB60_GPIO_KEYIN(x) (JZ_GPIO_PORTD(18) + (x)) @@ -466,27 +464,27 @@ static unsigned long pin_cfg_bias_disable[] = { static struct pinctrl_map pin_map[] __initdata = { /* NAND pin configuration */ PIN_MAP_MUX_GROUP_DEFAULT("jz4740-nand", - "10010000.jz4740-pinctrl", "nand", "nand-cs1"), + "10010000.pin-controller", "nand-cs1", "nand"), /* fbdev pin configuration */ PIN_MAP_MUX_GROUP("jz4740-fb", PINCTRL_STATE_DEFAULT, - "10010000.jz4740-pinctrl", "lcd", "lcd-8bit"), + "10010000.pin-controller", "lcd-8bit", "lcd"), PIN_MAP_MUX_GROUP("jz4740-fb", PINCTRL_STATE_SLEEP, - "10010000.jz4740-pinctrl", "lcd", "lcd-no-pins"), + "10010000.pin-controller", "lcd-no-pins", "lcd"), /* MMC pin configuration */ PIN_MAP_MUX_GROUP_DEFAULT("jz4740-mmc.0", - "10010000.jz4740-pinctrl", "mmc", "mmc-1bit"), + "10010000.pin-controller", "mmc-1bit", "mmc"), PIN_MAP_MUX_GROUP_DEFAULT("jz4740-mmc.0", - "10010000.jz4740-pinctrl", "mmc", "mmc-4bit"), + "10010000.pin-controller", "mmc-4bit", "mmc"), PIN_MAP_CONFIGS_PIN_DEFAULT("jz4740-mmc.0", - "10010000.jz4740-pinctrl", "PD0", pin_cfg_bias_disable), + "10010000.pin-controller", "PD0", pin_cfg_bias_disable), PIN_MAP_CONFIGS_PIN_DEFAULT("jz4740-mmc.0", - "10010000.jz4740-pinctrl", "PD2", pin_cfg_bias_disable), + "10010000.pin-controller", "PD2", pin_cfg_bias_disable), /* PWM pin configuration */ PIN_MAP_MUX_GROUP_DEFAULT("jz4740-pwm", - "10010000.jz4740-pinctrl", "pwm4", "pwm4"), + "10010000.pin-controller", "pwm4", "pwm4"), }; diff --git a/arch/mips/jz4740/platform.c b/arch/mips/jz4740/platform.c index 4b89abb17950..c74c99f5951d 100644 --- a/arch/mips/jz4740/platform.c +++ b/arch/mips/jz4740/platform.c @@ -21,8 +21,6 @@ #include <linux/serial_core.h> #include <linux/serial_8250.h> -#include "clock.h" - /* USB Device Controller */ struct platform_device jz4740_udc_xceiv_device = { .name = "usb_phy_generic", diff --git a/arch/mips/jz4740/pm.c b/arch/mips/jz4740/pm.c index bbdd2b801e6e..f9b551f01f42 100644 --- a/arch/mips/jz4740/pm.c +++ b/arch/mips/jz4740/pm.c @@ -9,21 +9,13 @@ #include <linux/delay.h> #include <linux/suspend.h> -#include <asm/mach-jz4740/clock.h> - static int jz4740_pm_enter(suspend_state_t state) { - jz4740_clock_suspend(); - - jz4740_clock_set_wait_mode(JZ4740_WAIT_MODE_SLEEP); - __asm__(".set\tmips3\n\t" "wait\n\t" ".set\tmips0"); - jz4740_clock_set_wait_mode(JZ4740_WAIT_MODE_IDLE); - jz4740_clock_resume(); return 0; } diff --git a/arch/mips/jz4740/time.c b/arch/mips/jz4740/time.c index a3260c754e65..cb768e560d8b 100644 --- a/arch/mips/jz4740/time.c +++ b/arch/mips/jz4740/time.c @@ -13,13 +13,10 @@ #include <linux/clockchips.h> #include <linux/sched_clock.h> -#include <asm/mach-jz4740/clock.h> #include <asm/mach-jz4740/irq.h> #include <asm/mach-jz4740/timer.h> #include <asm/time.h> -#include "clock.h" - #define TIMER_CLOCKEVENT 0 #define TIMER_CLOCKSOURCE 1 diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c index 4b5e1f2bfbce..2625232bfe52 100644 --- a/arch/mips/kernel/ftrace.c +++ b/arch/mips/kernel/ftrace.c @@ -333,20 +333,21 @@ void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, return; /* - * "parent_ra_addr" is the stack address saved the return address of - * the caller of _mcount. + * "parent_ra_addr" is the stack address where the return address of + * the caller of _mcount is saved. * - * if the gcc < 4.5, a leaf function does not save the return address - * in the stack address, so, we "emulate" one in _mcount's stack space, - * and hijack it directly, but for a non-leaf function, it save the - * return address to the its own stack space, we can not hijack it - * directly, but need to find the real stack address, - * ftrace_get_parent_addr() does it! + * If gcc < 4.5, a leaf function does not save the return address + * in the stack address, so we "emulate" one in _mcount's stack space, + * and hijack it directly. + * For a non-leaf function, it does save the return address to its own + * stack space, so we can not hijack it directly, but need to find the + * real stack address, which is done by ftrace_get_parent_addr(). * - * if gcc>= 4.5, with the new -mmcount-ra-address option, for a + * If gcc >= 4.5, with the new -mmcount-ra-address option, for a * non-leaf function, the location of the return address will be saved - * to $12 for us, and for a leaf function, only put a zero into $12. we - * do it in ftrace_graph_caller of mcount.S. + * to $12 for us. + * For a leaf function, it just puts a zero into $12, so we handle + * it in ftrace_graph_caller() of mcount.S. */ /* old_parent_ra = *parent_ra_addr; */ diff --git a/arch/mips/kernel/kprobes.c b/arch/mips/kernel/kprobes.c index 81ba1d3c367c..6cfae2411c04 100644 --- a/arch/mips/kernel/kprobes.c +++ b/arch/mips/kernel/kprobes.c @@ -398,7 +398,7 @@ out: return 1; } -static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr) +int kprobe_fault_handler(struct pt_regs *regs, int trapnr) { struct kprobe *cur = kprobe_running(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index e0ebaa0a333e..a3e2da8391ea 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -790,15 +790,19 @@ static void reset_counters(void *arg) case 4: mipsxx_pmu_write_control(3, 0); mipspmu.write_counter(3, 0); + /* fall through */ case 3: mipsxx_pmu_write_control(2, 0); mipspmu.write_counter(2, 0); + /* fall through */ case 2: mipsxx_pmu_write_control(1, 0); mipspmu.write_counter(1, 0); + /* fall through */ case 1: mipsxx_pmu_write_control(0, 0); mipspmu.write_counter(0, 0); + /* fall through */ } } @@ -1380,7 +1384,7 @@ static int mipsxx_pmu_handle_shared_irq(void) struct perf_sample_data data; unsigned int counters = mipspmu.num_counters; u64 counter; - int handled = IRQ_NONE; + int n, handled = IRQ_NONE; struct pt_regs *regs; if (cpu_has_perf_cntr_intr_bit && !(read_c0_cause() & CAUSEF_PCI)) @@ -1401,20 +1405,16 @@ static int mipsxx_pmu_handle_shared_irq(void) perf_sample_data_init(&data, 0, 0); - switch (counters) { -#define HANDLE_COUNTER(n) \ - case n + 1: \ - if (test_bit(n, cpuc->used_mask)) { \ - counter = mipspmu.read_counter(n); \ - if (counter & mipspmu.overflow) { \ - handle_associated_event(cpuc, n, &data, regs); \ - handled = IRQ_HANDLED; \ - } \ - } - HANDLE_COUNTER(3) - HANDLE_COUNTER(2) - HANDLE_COUNTER(1) - HANDLE_COUNTER(0) + for (n = counters - 1; n >= 0; n--) { + if (!test_bit(n, cpuc->used_mask)) + continue; + + counter = mipspmu.read_counter(n); + if (!(counter & mipspmu.overflow)) + continue; + + handle_associated_event(cpuc, n, &data, regs); + handled = IRQ_HANDLED; } #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS diff --git a/arch/mips/lantiq/irq.c b/arch/mips/lantiq/irq.c index cfd87e662fcf..115b417dfb8e 100644 --- a/arch/mips/lantiq/irq.c +++ b/arch/mips/lantiq/irq.c @@ -20,13 +20,13 @@ #include <irq.h> /* register definitions - internal irqs */ -#define LTQ_ICU_IM0_ISR 0x0000 -#define LTQ_ICU_IM0_IER 0x0008 -#define LTQ_ICU_IM0_IOSR 0x0010 -#define LTQ_ICU_IM0_IRSR 0x0018 -#define LTQ_ICU_IM0_IMR 0x0020 -#define LTQ_ICU_IM1_ISR 0x0028 -#define LTQ_ICU_OFFSET (LTQ_ICU_IM1_ISR - LTQ_ICU_IM0_ISR) +#define LTQ_ICU_ISR 0x0000 +#define LTQ_ICU_IER 0x0008 +#define LTQ_ICU_IOSR 0x0010 +#define LTQ_ICU_IRSR 0x0018 +#define LTQ_ICU_IMR 0x0020 + +#define LTQ_ICU_IM_SIZE 0x28 /* register definitions - external irqs */ #define LTQ_EIU_EXIN_C 0x0000 @@ -46,24 +46,25 @@ */ #define LTQ_ICU_EBU_IRQ 22 -#define ltq_icu_w32(m, x, y) ltq_w32((x), ltq_icu_membase[m] + (y)) -#define ltq_icu_r32(m, x) ltq_r32(ltq_icu_membase[m] + (x)) +#define ltq_icu_w32(vpe, m, x, y) \ + ltq_w32((x), ltq_icu_membase[vpe] + m*LTQ_ICU_IM_SIZE + (y)) + +#define ltq_icu_r32(vpe, m, x) \ + ltq_r32(ltq_icu_membase[vpe] + m*LTQ_ICU_IM_SIZE + (x)) #define ltq_eiu_w32(x, y) ltq_w32((x), ltq_eiu_membase + (y)) #define ltq_eiu_r32(x) ltq_r32(ltq_eiu_membase + (x)) -/* our 2 ipi interrupts for VSMP */ -#define MIPS_CPU_IPI_RESCHED_IRQ 0 -#define MIPS_CPU_IPI_CALL_IRQ 1 - /* we have a cascade of 8 irqs */ #define MIPS_CPU_IRQ_CASCADE 8 static int exin_avail; static u32 ltq_eiu_irq[MAX_EIU]; -static void __iomem *ltq_icu_membase[MAX_IM]; +static void __iomem *ltq_icu_membase[NR_CPUS]; static void __iomem *ltq_eiu_membase; static struct irq_domain *ltq_domain; +static DEFINE_SPINLOCK(ltq_eiu_lock); +static DEFINE_RAW_SPINLOCK(ltq_icu_lock); static int ltq_perfcount_irq; int ltq_eiu_get_irq(int exin) @@ -75,49 +76,84 @@ int ltq_eiu_get_irq(int exin) void ltq_disable_irq(struct irq_data *d) { - u32 ier = LTQ_ICU_IM0_IER; - int offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; - int im = offset / INT_NUM_IM_OFFSET; + unsigned long offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; + unsigned long im = offset / INT_NUM_IM_OFFSET; + unsigned long flags; + int vpe; offset %= INT_NUM_IM_OFFSET; - ltq_icu_w32(im, ltq_icu_r32(im, ier) & ~BIT(offset), ier); + + raw_spin_lock_irqsave(<q_icu_lock, flags); + for_each_present_cpu(vpe) { + ltq_icu_w32(vpe, im, + ltq_icu_r32(vpe, im, LTQ_ICU_IER) & ~BIT(offset), + LTQ_ICU_IER); + } + raw_spin_unlock_irqrestore(<q_icu_lock, flags); } void ltq_mask_and_ack_irq(struct irq_data *d) { - u32 ier = LTQ_ICU_IM0_IER; - u32 isr = LTQ_ICU_IM0_ISR; - int offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; - int im = offset / INT_NUM_IM_OFFSET; + unsigned long offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; + unsigned long im = offset / INT_NUM_IM_OFFSET; + unsigned long flags; + int vpe; offset %= INT_NUM_IM_OFFSET; - ltq_icu_w32(im, ltq_icu_r32(im, ier) & ~BIT(offset), ier); - ltq_icu_w32(im, BIT(offset), isr); + + raw_spin_lock_irqsave(<q_icu_lock, flags); + for_each_present_cpu(vpe) { + ltq_icu_w32(vpe, im, + ltq_icu_r32(vpe, im, LTQ_ICU_IER) & ~BIT(offset), + LTQ_ICU_IER); + ltq_icu_w32(vpe, im, BIT(offset), LTQ_ICU_ISR); + } + raw_spin_unlock_irqrestore(<q_icu_lock, flags); } static void ltq_ack_irq(struct irq_data *d) { - u32 isr = LTQ_ICU_IM0_ISR; - int offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; - int im = offset / INT_NUM_IM_OFFSET; + unsigned long offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; + unsigned long im = offset / INT_NUM_IM_OFFSET; + unsigned long flags; + int vpe; offset %= INT_NUM_IM_OFFSET; - ltq_icu_w32(im, BIT(offset), isr); + + raw_spin_lock_irqsave(<q_icu_lock, flags); + for_each_present_cpu(vpe) { + ltq_icu_w32(vpe, im, BIT(offset), LTQ_ICU_ISR); + } + raw_spin_unlock_irqrestore(<q_icu_lock, flags); } void ltq_enable_irq(struct irq_data *d) { - u32 ier = LTQ_ICU_IM0_IER; - int offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; - int im = offset / INT_NUM_IM_OFFSET; + unsigned long offset = d->hwirq - MIPS_CPU_IRQ_CASCADE; + unsigned long im = offset / INT_NUM_IM_OFFSET; + unsigned long flags; + int vpe; offset %= INT_NUM_IM_OFFSET; - ltq_icu_w32(im, ltq_icu_r32(im, ier) | BIT(offset), ier); + + vpe = cpumask_first(irq_data_get_effective_affinity_mask(d)); + + /* This shouldn't be even possible, maybe during CPU hotplug spam */ + if (unlikely(vpe >= nr_cpu_ids)) + vpe = smp_processor_id(); + + raw_spin_lock_irqsave(<q_icu_lock, flags); + + ltq_icu_w32(vpe, im, ltq_icu_r32(vpe, im, LTQ_ICU_IER) | BIT(offset), + LTQ_ICU_IER); + + raw_spin_unlock_irqrestore(<q_icu_lock, flags); } static int ltq_eiu_settype(struct irq_data *d, unsigned int type) { int i; + unsigned long flags; for (i = 0; i < exin_avail; i++) { if (d->hwirq == ltq_eiu_irq[i]) { @@ -154,8 +190,11 @@ static int ltq_eiu_settype(struct irq_data *d, unsigned int type) if (edge) irq_set_handler(d->hwirq, handle_edge_irq); - ltq_eiu_w32(ltq_eiu_r32(LTQ_EIU_EXIN_C) | - (val << (i * 4)), LTQ_EIU_EXIN_C); + spin_lock_irqsave(<q_eiu_lock, flags); + ltq_eiu_w32((ltq_eiu_r32(LTQ_EIU_EXIN_C) & + (~(7 << (i * 4)))) | (val << (i * 4)), + LTQ_EIU_EXIN_C); + spin_unlock_irqrestore(<q_eiu_lock, flags); } } @@ -199,6 +238,21 @@ static void ltq_shutdown_eiu_irq(struct irq_data *d) } } +#if defined(CONFIG_SMP) +static int ltq_icu_irq_set_affinity(struct irq_data *d, + const struct cpumask *cpumask, bool force) +{ + struct cpumask tmask; + + if (!cpumask_and(&tmask, cpumask, cpu_online_mask)) + return -EINVAL; + + irq_data_update_effective_affinity(d, &tmask); + + return IRQ_SET_MASK_OK; +} +#endif + static struct irq_chip ltq_irq_type = { .name = "icu", .irq_enable = ltq_enable_irq, @@ -207,6 +261,9 @@ static struct irq_chip ltq_irq_type = { .irq_ack = ltq_ack_irq, .irq_mask = ltq_disable_irq, .irq_mask_ack = ltq_mask_and_ack_irq, +#if defined(CONFIG_SMP) + .irq_set_affinity = ltq_icu_irq_set_affinity, +#endif }; static struct irq_chip ltq_eiu_type = { @@ -220,15 +277,19 @@ static struct irq_chip ltq_eiu_type = { .irq_mask = ltq_disable_irq, .irq_mask_ack = ltq_mask_and_ack_irq, .irq_set_type = ltq_eiu_settype, +#if defined(CONFIG_SMP) + .irq_set_affinity = ltq_icu_irq_set_affinity, +#endif }; static void ltq_hw_irq_handler(struct irq_desc *desc) { - int module = irq_desc_get_irq(desc) - 2; + unsigned int module = irq_desc_get_irq(desc) - 2; u32 irq; - int hwirq; + irq_hw_number_t hwirq; + int vpe = smp_processor_id(); - irq = ltq_icu_r32(module, LTQ_ICU_IM0_IOSR); + irq = ltq_icu_r32(vpe, module, LTQ_ICU_IOSR); if (irq == 0) return; @@ -249,6 +310,7 @@ static void ltq_hw_irq_handler(struct irq_desc *desc) static int icu_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) { struct irq_chip *chip = <q_irq_type; + struct irq_data *data; int i; if (hw < MIPS_CPU_IRQ_CASCADE) @@ -258,6 +320,10 @@ static int icu_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) if (hw == ltq_eiu_irq[i]) chip = <q_eiu_type; + data = irq_get_irq_data(irq); + + irq_data_update_effective_affinity(data, cpumask_of(0)); + irq_set_chip_and_handler(irq, chip, handle_level_irq); return 0; @@ -272,28 +338,37 @@ int __init icu_of_init(struct device_node *node, struct device_node *parent) { struct device_node *eiu_node; struct resource res; - int i, ret; + int i, ret, vpe; - for (i = 0; i < MAX_IM; i++) { - if (of_address_to_resource(node, i, &res)) - panic("Failed to get icu memory range"); + /* load register regions of available ICUs */ + for_each_possible_cpu(vpe) { + if (of_address_to_resource(node, vpe, &res)) + panic("Failed to get icu%i memory range", vpe); if (!request_mem_region(res.start, resource_size(&res), res.name)) - pr_err("Failed to request icu memory"); + pr_err("Failed to request icu%i memory\n", vpe); - ltq_icu_membase[i] = ioremap_nocache(res.start, + ltq_icu_membase[vpe] = ioremap_nocache(res.start, resource_size(&res)); - if (!ltq_icu_membase[i]) - panic("Failed to remap icu memory"); + + if (!ltq_icu_membase[vpe]) + panic("Failed to remap icu%i memory", vpe); } /* turn off all irqs by default */ - for (i = 0; i < MAX_IM; i++) { - /* make sure all irqs are turned off by default */ - ltq_icu_w32(i, 0, LTQ_ICU_IM0_IER); - /* clear all possibly pending interrupts */ - ltq_icu_w32(i, ~0, LTQ_ICU_IM0_ISR); + for_each_possible_cpu(vpe) { + for (i = 0; i < MAX_IM; i++) { + /* make sure all irqs are turned off by default */ + ltq_icu_w32(vpe, i, 0, LTQ_ICU_IER); + + /* clear all possibly pending interrupts */ + ltq_icu_w32(vpe, i, ~0, LTQ_ICU_ISR); + ltq_icu_w32(vpe, i, ~0, LTQ_ICU_IMR); + + /* clear resend */ + ltq_icu_w32(vpe, i, 0, LTQ_ICU_IRSR); + } } mips_cpu_irq_init(); @@ -347,7 +422,7 @@ unsigned int get_c0_compare_int(void) return CP0_LEGACY_COMPARE_IRQ; } -static struct of_device_id __initdata of_irq_ids[] = { +static const struct of_device_id of_irq_ids[] __initconst = { { .compatible = "lantiq,icu", .data = icu_of_init }, {}, }; diff --git a/arch/nds32/include/asm/syscall.h b/arch/nds32/include/asm/syscall.h index 899b2fb4b52f..7b5180d78e20 100644 --- a/arch/nds32/include/asm/syscall.h +++ b/arch/nds32/include/asm/syscall.h @@ -26,7 +26,8 @@ struct pt_regs; * * It's only valid to call this when @task is known to be blocked. */ -int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) +static inline int +syscall_get_nr(struct task_struct *task, struct pt_regs *regs) { return regs->syscallno; } @@ -47,7 +48,8 @@ int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) * system call instruction. This may not be the same as what the * register state looked like at system call entry tracing. */ -void syscall_rollback(struct task_struct *task, struct pt_regs *regs) +static inline void +syscall_rollback(struct task_struct *task, struct pt_regs *regs) { regs->uregs[0] = regs->orig_r0; } @@ -62,7 +64,8 @@ void syscall_rollback(struct task_struct *task, struct pt_regs *regs) * It's only valid to call this when @task is stopped for tracing on exit * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. */ -long syscall_get_error(struct task_struct *task, struct pt_regs *regs) +static inline long +syscall_get_error(struct task_struct *task, struct pt_regs *regs) { unsigned long error = regs->uregs[0]; return IS_ERR_VALUE(error) ? error : 0; @@ -79,7 +82,8 @@ long syscall_get_error(struct task_struct *task, struct pt_regs *regs) * It's only valid to call this when @task is stopped for tracing on exit * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. */ -long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) +static inline long +syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { return regs->uregs[0]; } @@ -99,8 +103,9 @@ long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) * It's only valid to call this when @task is stopped for tracing on exit * from a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. */ -void syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, - int error, long val) +static inline void +syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, + int error, long val) { regs->uregs[0] = (long)error ? error : val; } @@ -118,8 +123,9 @@ void syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, * entry to a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. */ #define SYSCALL_MAX_ARGS 6 -void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned long *args) +static inline void +syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, + unsigned long *args) { args[0] = regs->orig_r0; args++; @@ -138,8 +144,9 @@ void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, * It's only valid to call this when @task is stopped for tracing on * entry to a system call, due to %TIF_SYSCALL_TRACE or %TIF_SYSCALL_AUDIT. */ -void syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, - const unsigned long *args) +static inline void +syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, + const unsigned long *args) { regs->orig_r0 = args[0]; args++; diff --git a/arch/parisc/include/asm/syscall.h b/arch/parisc/include/asm/syscall.h index 80757e43cf2c..00b127a5e09b 100644 --- a/arch/parisc/include/asm/syscall.h +++ b/arch/parisc/include/asm/syscall.h @@ -29,6 +29,13 @@ static inline void syscall_get_arguments(struct task_struct *tsk, args[0] = regs->gr[26]; } +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + unsigned long error = regs->gr[28]; + return IS_ERR_VALUE(error) ? error : 0; +} + static inline long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index f516796dd819..d8dcd8820369 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -129,6 +129,7 @@ config PPC select ARCH_HAS_MMIOWB if PPC64 select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_PMEM_API if PPC64 + select ARCH_HAS_PTE_DEVMAP if PPC_BOOK3S_64 select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_MEMBARRIER_CALLBACKS select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC64 @@ -136,7 +137,6 @@ config PPC select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UACCESS_FLUSHCACHE if PPC64 select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_ZONE_DEVICE if PPC_BOOK3S_64 select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 62e6ea0a7650..8308f32e9782 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -90,7 +90,6 @@ #define _PAGE_SOFT_DIRTY _RPAGE_SW3 /* software: software dirty tracking */ #define _PAGE_SPECIAL _RPAGE_SW2 /* software: special page */ #define _PAGE_DEVMAP _RPAGE_SW1 /* software: ZONE_DEVICE page */ -#define __HAVE_ARCH_PTE_DEVMAP /* * Drivers request for cache inhibited pte mapping using _PAGE_NO_CACHE diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h index 81abcf6a737b..38d62acfdce7 100644 --- a/arch/powerpc/include/asm/syscall.h +++ b/arch/powerpc/include/asm/syscall.h @@ -35,6 +35,16 @@ static inline void syscall_rollback(struct task_struct *task, regs->gpr[3] = regs->orig_gpr3; } +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + /* + * If the system call failed, + * regs->gpr[3] contains a positive ERRORCODE. + */ + return (regs->ccr & 0x10000000UL) ? -regs->gpr[3] : 0; +} + static inline long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { diff --git a/arch/powerpc/include/uapi/asm/mman.h b/arch/powerpc/include/uapi/asm/mman.h index 65065ce32814..c0c737215b00 100644 --- a/arch/powerpc/include/uapi/asm/mman.h +++ b/arch/powerpc/include/uapi/asm/mman.h @@ -21,15 +21,11 @@ #define MAP_DENYWRITE 0x0800 /* ETXTBSY */ #define MAP_EXECUTABLE 0x1000 /* mark it as an executable */ + #define MCL_CURRENT 0x2000 /* lock all currently mapped pages */ #define MCL_FUTURE 0x4000 /* lock all additions to address space */ #define MCL_ONFAULT 0x8000 /* lock all pages that are faulted in */ -#define MAP_POPULATE 0x8000 /* populate (prefault) pagetables */ -#define MAP_NONBLOCK 0x10000 /* do not block on IO */ -#define MAP_STACK 0x20000 /* give out an address that is best suited for process/thread stacks */ -#define MAP_HUGETLB 0x40000 /* create a huge page mapping */ - /* Override any generic PKEY permission defines */ #define PKEY_DISABLE_EXECUTE 0x4 #undef PKEY_ACCESS_MASK diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 5bf05cc774e2..e99a14798ab0 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -19,6 +19,7 @@ #include <linux/anon_inodes.h> #include <linux/iommu.h> #include <linux/file.h> +#include <linux/mm.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> @@ -45,43 +46,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages) return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE; } -static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc) -{ - long ret = 0; - - if (!current || !current->mm) - return ret; /* process exited */ - - down_write(¤t->mm->mmap_sem); - - if (inc) { - unsigned long locked, lock_limit; - - locked = current->mm->locked_vm + stt_pages; - lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked > lock_limit && !capable(CAP_IPC_LOCK)) - ret = -ENOMEM; - else - current->mm->locked_vm += stt_pages; - } else { - if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm)) - stt_pages = current->mm->locked_vm; - - current->mm->locked_vm -= stt_pages; - } - - pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid, - inc ? '+' : '-', - stt_pages << PAGE_SHIFT, - current->mm->locked_vm << PAGE_SHIFT, - rlimit(RLIMIT_MEMLOCK), - ret ? " - exceeded" : ""); - - up_write(¤t->mm->mmap_sem); - - return ret; -} - static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head) { struct kvmppc_spapr_tce_iommu_table *stit = container_of(head, @@ -291,7 +255,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp) kvm_put_kvm(stt->kvm); - kvmppc_account_memlimit( + account_locked_vm(current->mm, kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false); call_rcu(&stt->rcu, release_spapr_tce_table); @@ -316,7 +280,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, return -EINVAL; npages = kvmppc_tce_pages(size); - ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true); + ret = account_locked_vm(current->mm, kvmppc_stt_pages(npages), true); if (ret) return ret; @@ -362,7 +326,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm, kfree(stt); fail_acct: - kvmppc_account_memlimit(kvmppc_stt_pages(npages), false); + account_locked_vm(current->mm, kvmppc_stt_pages(npages), false); return ret; } diff --git a/arch/powerpc/mm/book3s64/iommu_api.c b/arch/powerpc/mm/book3s64/iommu_api.c index 90ee3a89722c..b056cae3388b 100644 --- a/arch/powerpc/mm/book3s64/iommu_api.c +++ b/arch/powerpc/mm/book3s64/iommu_api.c @@ -14,6 +14,7 @@ #include <linux/hugetlb.h> #include <linux/swap.h> #include <linux/sizes.h> +#include <linux/mm.h> #include <asm/mmu_context.h> #include <asm/pte-walk.h> #include <linux/mm_inline.h> @@ -46,40 +47,6 @@ struct mm_iommu_table_group_mem_t { u64 dev_hpa; /* Device memory base address */ }; -static long mm_iommu_adjust_locked_vm(struct mm_struct *mm, - unsigned long npages, bool incr) -{ - long ret = 0, locked, lock_limit; - - if (!npages) - return 0; - - down_write(&mm->mmap_sem); - - if (incr) { - locked = mm->locked_vm + npages; - lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked > lock_limit && !capable(CAP_IPC_LOCK)) - ret = -ENOMEM; - else - mm->locked_vm += npages; - } else { - if (WARN_ON_ONCE(npages > mm->locked_vm)) - npages = mm->locked_vm; - mm->locked_vm -= npages; - } - - pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n", - current ? current->pid : 0, - incr ? '+' : '-', - npages << PAGE_SHIFT, - mm->locked_vm << PAGE_SHIFT, - rlimit(RLIMIT_MEMLOCK)); - up_write(&mm->mmap_sem); - - return ret; -} - bool mm_iommu_preregistered(struct mm_struct *mm) { return !list_empty(&mm->context.iommu_group_mem_list); @@ -96,7 +63,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua, unsigned long entry, chunk; if (dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) { - ret = mm_iommu_adjust_locked_vm(mm, entries, true); + ret = account_locked_vm(mm, entries, true); if (ret) return ret; @@ -211,7 +178,7 @@ free_exit: kfree(mem); unlock_exit: - mm_iommu_adjust_locked_vm(mm, locked_entries, false); + account_locked_vm(mm, locked_entries, false); return ret; } @@ -311,7 +278,7 @@ long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem) unlock_exit: mutex_unlock(&mem_list_mutex); - mm_iommu_adjust_locked_vm(mm, unlock_entries, false); + account_locked_vm(mm, unlock_entries, false); return ret; } diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 65c2ba1e1783..b4ca9e95e678 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1237,3 +1237,8 @@ int radix__ioremap_range(unsigned long ea, phys_addr_t pa, unsigned long size, return 0; } } + +int __init arch_ioremap_p4d_supported(void) +{ + return 0; +} diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index d989592b6fc8..8432c281de92 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -42,26 +42,6 @@ #include <asm/debug.h> #include <asm/kup.h> -static inline bool notify_page_fault(struct pt_regs *regs) -{ - bool ret = false; - -#ifdef CONFIG_KPROBES - /* kprobe_running() needs smp_processor_id() */ - if (!user_mode(regs)) { - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, 11)) - ret = true; - preempt_enable(); - } -#endif /* CONFIG_KPROBES */ - - if (unlikely(debugger_fault_handler(regs))) - ret = true; - - return ret; -} - /* * Check whether the instruction inst is a store using * an update addressing form which will update r1. @@ -461,8 +441,9 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, int is_write = page_fault_is_write(error_code); vm_fault_t fault, major = 0; bool must_retry = false; + bool kprobe_fault = kprobe_page_fault(regs, 11); - if (notify_page_fault(regs)) + if (unlikely(debugger_fault_handler(regs) || kprobe_fault)) return 0; if (unlikely(page_fault_is_bad(error_code))) { diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index 0cf6b53587db..60f907516335 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -8,27 +8,27 @@ #ifndef __ASM_CTL_REG_H #define __ASM_CTL_REG_H -#include <linux/const.h> - -#define CR0_CLOCK_COMPARATOR_SIGN _BITUL(63 - 10) -#define CR0_EMERGENCY_SIGNAL_SUBMASK _BITUL(63 - 49) -#define CR0_EXTERNAL_CALL_SUBMASK _BITUL(63 - 50) -#define CR0_CLOCK_COMPARATOR_SUBMASK _BITUL(63 - 52) -#define CR0_CPU_TIMER_SUBMASK _BITUL(63 - 53) -#define CR0_SERVICE_SIGNAL_SUBMASK _BITUL(63 - 54) -#define CR0_UNUSED_56 _BITUL(63 - 56) -#define CR0_INTERRUPT_KEY_SUBMASK _BITUL(63 - 57) -#define CR0_MEASUREMENT_ALERT_SUBMASK _BITUL(63 - 58) - -#define CR2_GUARDED_STORAGE _BITUL(63 - 59) - -#define CR14_UNUSED_32 _BITUL(63 - 32) -#define CR14_UNUSED_33 _BITUL(63 - 33) -#define CR14_CHANNEL_REPORT_SUBMASK _BITUL(63 - 35) -#define CR14_RECOVERY_SUBMASK _BITUL(63 - 36) -#define CR14_DEGRADATION_SUBMASK _BITUL(63 - 37) -#define CR14_EXTERNAL_DAMAGE_SUBMASK _BITUL(63 - 38) -#define CR14_WARNING_SUBMASK _BITUL(63 - 39) +#include <linux/bits.h> + +#define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) +#define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) +#define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) +#define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) +#define CR0_CPU_TIMER_SUBMASK BIT(63 - 53) +#define CR0_SERVICE_SIGNAL_SUBMASK BIT(63 - 54) +#define CR0_UNUSED_56 BIT(63 - 56) +#define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) +#define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) + +#define CR2_GUARDED_STORAGE BIT(63 - 59) + +#define CR14_UNUSED_32 BIT(63 - 32) +#define CR14_UNUSED_33 BIT(63 - 33) +#define CR14_CHANNEL_REPORT_SUBMASK BIT(63 - 35) +#define CR14_RECOVERY_SUBMASK BIT(63 - 36) +#define CR14_DEGRADATION_SUBMASK BIT(63 - 37) +#define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(63 - 38) +#define CR14_WARNING_SUBMASK BIT(63 - 39) #ifndef __ASSEMBLY__ diff --git a/arch/s390/include/asm/nmi.h b/arch/s390/include/asm/nmi.h index 1e5dc4537bf2..b160da8fa14b 100644 --- a/arch/s390/include/asm/nmi.h +++ b/arch/s390/include/asm/nmi.h @@ -12,7 +12,7 @@ #ifndef _ASM_S390_NMI_H #define _ASM_S390_NMI_H -#include <linux/const.h> +#include <linux/bits.h> #include <linux/types.h> #define MCIC_SUBCLASS_MASK (1ULL<<63 | 1ULL<<62 | 1ULL<<61 | \ @@ -20,15 +20,15 @@ 1ULL<<55 | 1ULL<<54 | 1ULL<<53 | \ 1ULL<<52 | 1ULL<<47 | 1ULL<<46 | \ 1ULL<<45 | 1ULL<<44) -#define MCCK_CODE_SYSTEM_DAMAGE _BITUL(63) -#define MCCK_CODE_EXT_DAMAGE _BITUL(63 - 5) -#define MCCK_CODE_CP _BITUL(63 - 9) -#define MCCK_CODE_CPU_TIMER_VALID _BITUL(63 - 46) -#define MCCK_CODE_PSW_MWP_VALID _BITUL(63 - 20) -#define MCCK_CODE_PSW_IA_VALID _BITUL(63 - 23) -#define MCCK_CODE_CR_VALID _BITUL(63 - 29) -#define MCCK_CODE_GS_VALID _BITUL(63 - 36) -#define MCCK_CODE_FC_VALID _BITUL(63 - 43) +#define MCCK_CODE_SYSTEM_DAMAGE BIT(63) +#define MCCK_CODE_EXT_DAMAGE BIT(63 - 5) +#define MCCK_CODE_CP BIT(63 - 9) +#define MCCK_CODE_CPU_TIMER_VALID BIT(63 - 46) +#define MCCK_CODE_PSW_MWP_VALID BIT(63 - 20) +#define MCCK_CODE_PSW_IA_VALID BIT(63 - 23) +#define MCCK_CODE_CR_VALID BIT(63 - 29) +#define MCCK_CODE_GS_VALID BIT(63 - 36) +#define MCCK_CODE_FC_VALID BIT(63 - 43) #ifndef __ASSEMBLY__ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 14883b1562e0..d56c519bc696 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -12,7 +12,7 @@ #ifndef __ASM_S390_PROCESSOR_H #define __ASM_S390_PROCESSOR_H -#include <linux/const.h> +#include <linux/bits.h> #define CIF_MCCK_PENDING 0 /* machine check handling is pending */ #define CIF_ASCE_PRIMARY 1 /* primary asce needs fixup / uaccess */ @@ -24,15 +24,15 @@ #define CIF_MCCK_GUEST 7 /* machine check happening in guest */ #define CIF_DEDICATED_CPU 8 /* this CPU is dedicated */ -#define _CIF_MCCK_PENDING _BITUL(CIF_MCCK_PENDING) -#define _CIF_ASCE_PRIMARY _BITUL(CIF_ASCE_PRIMARY) -#define _CIF_ASCE_SECONDARY _BITUL(CIF_ASCE_SECONDARY) -#define _CIF_NOHZ_DELAY _BITUL(CIF_NOHZ_DELAY) -#define _CIF_FPU _BITUL(CIF_FPU) -#define _CIF_IGNORE_IRQ _BITUL(CIF_IGNORE_IRQ) -#define _CIF_ENABLED_WAIT _BITUL(CIF_ENABLED_WAIT) -#define _CIF_MCCK_GUEST _BITUL(CIF_MCCK_GUEST) -#define _CIF_DEDICATED_CPU _BITUL(CIF_DEDICATED_CPU) +#define _CIF_MCCK_PENDING BIT(CIF_MCCK_PENDING) +#define _CIF_ASCE_PRIMARY BIT(CIF_ASCE_PRIMARY) +#define _CIF_ASCE_SECONDARY BIT(CIF_ASCE_SECONDARY) +#define _CIF_NOHZ_DELAY BIT(CIF_NOHZ_DELAY) +#define _CIF_FPU BIT(CIF_FPU) +#define _CIF_IGNORE_IRQ BIT(CIF_IGNORE_IRQ) +#define _CIF_ENABLED_WAIT BIT(CIF_ENABLED_WAIT) +#define _CIF_MCCK_GUEST BIT(CIF_MCCK_GUEST) +#define _CIF_DEDICATED_CPU BIT(CIF_DEDICATED_CPU) #ifndef __ASSEMBLY__ diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 6f70d81c40f2..f009a13afe71 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -7,7 +7,7 @@ #ifndef _S390_PTRACE_H #define _S390_PTRACE_H -#include <linux/const.h> +#include <linux/bits.h> #include <uapi/asm/ptrace.h> #define PIF_SYSCALL 0 /* inside a system call */ @@ -15,10 +15,10 @@ #define PIF_SYSCALL_RESTART 2 /* restart the current system call */ #define PIF_GUEST_FAULT 3 /* indicates program check in sie64a */ -#define _PIF_SYSCALL _BITUL(PIF_SYSCALL) -#define _PIF_PER_TRAP _BITUL(PIF_PER_TRAP) -#define _PIF_SYSCALL_RESTART _BITUL(PIF_SYSCALL_RESTART) -#define _PIF_GUEST_FAULT _BITUL(PIF_GUEST_FAULT) +#define _PIF_SYSCALL BIT(PIF_SYSCALL) +#define _PIF_PER_TRAP BIT(PIF_PER_TRAP) +#define _PIF_SYSCALL_RESTART BIT(PIF_SYSCALL_RESTART) +#define _PIF_GUEST_FAULT BIT(PIF_GUEST_FAULT) #ifndef __ASSEMBLY__ diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 925889d360c1..82deb8fc8319 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -6,7 +6,7 @@ #ifndef _ASM_S390_SETUP_H #define _ASM_S390_SETUP_H -#include <linux/const.h> +#include <linux/bits.h> #include <uapi/asm/setup.h> #define EP_OFFSET 0x10008 @@ -21,25 +21,25 @@ * Machine features detected in early.c */ -#define MACHINE_FLAG_VM _BITUL(0) -#define MACHINE_FLAG_KVM _BITUL(1) -#define MACHINE_FLAG_LPAR _BITUL(2) -#define MACHINE_FLAG_DIAG9C _BITUL(3) -#define MACHINE_FLAG_ESOP _BITUL(4) -#define MACHINE_FLAG_IDTE _BITUL(5) -#define MACHINE_FLAG_DIAG44 _BITUL(6) -#define MACHINE_FLAG_EDAT1 _BITUL(7) -#define MACHINE_FLAG_EDAT2 _BITUL(8) -#define MACHINE_FLAG_TOPOLOGY _BITUL(10) -#define MACHINE_FLAG_TE _BITUL(11) -#define MACHINE_FLAG_TLB_LC _BITUL(12) -#define MACHINE_FLAG_VX _BITUL(13) -#define MACHINE_FLAG_TLB_GUEST _BITUL(14) -#define MACHINE_FLAG_NX _BITUL(15) -#define MACHINE_FLAG_GS _BITUL(16) -#define MACHINE_FLAG_SCC _BITUL(17) - -#define LPP_MAGIC _BITUL(31) +#define MACHINE_FLAG_VM BIT(0) +#define MACHINE_FLAG_KVM BIT(1) +#define MACHINE_FLAG_LPAR BIT(2) +#define MACHINE_FLAG_DIAG9C BIT(3) +#define MACHINE_FLAG_ESOP BIT(4) +#define MACHINE_FLAG_IDTE BIT(5) +#define MACHINE_FLAG_DIAG44 BIT(6) +#define MACHINE_FLAG_EDAT1 BIT(7) +#define MACHINE_FLAG_EDAT2 BIT(8) +#define MACHINE_FLAG_TOPOLOGY BIT(10) +#define MACHINE_FLAG_TE BIT(11) +#define MACHINE_FLAG_TLB_LC BIT(12) +#define MACHINE_FLAG_VX BIT(13) +#define MACHINE_FLAG_TLB_GUEST BIT(14) +#define MACHINE_FLAG_NX BIT(15) +#define MACHINE_FLAG_GS BIT(16) +#define MACHINE_FLAG_SCC BIT(17) + +#define LPP_MAGIC BIT(31) #define LPP_PID_MASK _AC(0xffffffff, UL) /* Offsets to entry points in kernel/head.S */ diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index ce4e17c9aad6..e582fbe59e20 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -8,7 +8,7 @@ #ifndef _ASM_THREAD_INFO_H #define _ASM_THREAD_INFO_H -#include <linux/const.h> +#include <linux/bits.h> /* * General size of kernel stacks @@ -82,21 +82,21 @@ void arch_setup_new_exec(void); #define TIF_SECCOMP 26 /* secure computing */ #define TIF_SYSCALL_TRACEPOINT 27 /* syscall tracepoint instrumentation */ -#define _TIF_NOTIFY_RESUME _BITUL(TIF_NOTIFY_RESUME) -#define _TIF_SIGPENDING _BITUL(TIF_SIGPENDING) -#define _TIF_NEED_RESCHED _BITUL(TIF_NEED_RESCHED) -#define _TIF_UPROBE _BITUL(TIF_UPROBE) -#define _TIF_GUARDED_STORAGE _BITUL(TIF_GUARDED_STORAGE) -#define _TIF_PATCH_PENDING _BITUL(TIF_PATCH_PENDING) -#define _TIF_ISOLATE_BP _BITUL(TIF_ISOLATE_BP) -#define _TIF_ISOLATE_BP_GUEST _BITUL(TIF_ISOLATE_BP_GUEST) - -#define _TIF_31BIT _BITUL(TIF_31BIT) -#define _TIF_SINGLE_STEP _BITUL(TIF_SINGLE_STEP) - -#define _TIF_SYSCALL_TRACE _BITUL(TIF_SYSCALL_TRACE) -#define _TIF_SYSCALL_AUDIT _BITUL(TIF_SYSCALL_AUDIT) -#define _TIF_SECCOMP _BITUL(TIF_SECCOMP) -#define _TIF_SYSCALL_TRACEPOINT _BITUL(TIF_SYSCALL_TRACEPOINT) +#define _TIF_NOTIFY_RESUME BIT(TIF_NOTIFY_RESUME) +#define _TIF_SIGPENDING BIT(TIF_SIGPENDING) +#define _TIF_NEED_RESCHED BIT(TIF_NEED_RESCHED) +#define _TIF_UPROBE BIT(TIF_UPROBE) +#define _TIF_GUARDED_STORAGE BIT(TIF_GUARDED_STORAGE) +#define _TIF_PATCH_PENDING BIT(TIF_PATCH_PENDING) +#define _TIF_ISOLATE_BP BIT(TIF_ISOLATE_BP) +#define _TIF_ISOLATE_BP_GUEST BIT(TIF_ISOLATE_BP_GUEST) + +#define _TIF_31BIT BIT(TIF_31BIT) +#define _TIF_SINGLE_STEP BIT(TIF_SINGLE_STEP) + +#define _TIF_SYSCALL_TRACE BIT(TIF_SYSCALL_TRACE) +#define _TIF_SYSCALL_AUDIT BIT(TIF_SYSCALL_AUDIT) +#define _TIF_SECCOMP BIT(TIF_SECCOMP) +#define _TIF_SYSCALL_TRACEPOINT BIT(TIF_SYSCALL_TRACEPOINT) #endif /* _ASM_THREAD_INFO_H */ diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 0ba174f779da..63507662828f 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -67,20 +67,6 @@ static int __init fault_init(void) } early_initcall(fault_init); -static inline int notify_page_fault(struct pt_regs *regs) -{ - int ret = 0; - - /* kprobe_running() needs smp_processor_id() */ - if (kprobes_built_in() && !user_mode(regs)) { - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, 14)) - ret = 1; - preempt_enable(); - } - return ret; -} - /* * Find out which address space caused the exception. */ @@ -412,7 +398,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) */ clear_pt_regs_flag(regs, PIF_PER_TRAP); - if (notify_page_fault(regs)) + if (kprobe_page_fault(regs, 14)) return 0; mm = tsk->mm; diff --git a/arch/sh/boards/Kconfig b/arch/sh/boards/Kconfig index b9a37057b77a..cee24c308337 100644 --- a/arch/sh/boards/Kconfig +++ b/arch/sh/boards/Kconfig @@ -8,27 +8,19 @@ config SH_ALPHA_BOARD bool config SH_DEVICE_TREE - bool "Board Described by Device Tree" + bool select OF select OF_EARLY_FLATTREE select TIMER_OF select COMMON_CLK select GENERIC_CALIBRATE_DELAY - help - Select Board Described by Device Tree to build a kernel that - does not hard-code any board-specific knowledge but instead uses - a device tree blob provided by the boot-loader. You must enable - drivers for any hardware you want to use separately. At this - time, only boards based on the open-hardware J-Core processors - have sufficient driver coverage to use this option; do not - select it if you are using original SuperH hardware. config SH_JCORE_SOC bool "J-Core SoC" - depends on SH_DEVICE_TREE && (CPU_SH2 || CPU_J2) + select SH_DEVICE_TREE select CLKSRC_JCORE_PIT select JCORE_AIC - default y if CPU_J2 + depends on CPU_J2 help Select this option to include drivers core components of the J-Core SoC, including interrupt controllers and timers. diff --git a/arch/sh/configs/ap325rxa_defconfig b/arch/sh/configs/ap325rxa_defconfig index 0ef3f1f9de5c..cc6e4ce53dac 100644 --- a/arch/sh/configs/ap325rxa_defconfig +++ b/arch/sh/configs/ap325rxa_defconfig @@ -28,7 +28,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_XFRM_MODE_TUNNEL is not set # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/apsh4a3a_defconfig b/arch/sh/configs/apsh4a3a_defconfig index 6c7cdc3beb28..530498f18990 100644 --- a/arch/sh/configs/apsh4a3a_defconfig +++ b/arch/sh/configs/apsh4a3a_defconfig @@ -29,7 +29,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_MTD=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/apsh4ad0a_defconfig b/arch/sh/configs/apsh4ad0a_defconfig index d0d9ebc7165b..6dd0da73ca5a 100644 --- a/arch/sh/configs/apsh4ad0a_defconfig +++ b/arch/sh/configs/apsh4ad0a_defconfig @@ -53,7 +53,6 @@ CONFIG_NET_KEY=y CONFIG_INET=y # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_MTD=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/cayman_defconfig b/arch/sh/configs/cayman_defconfig index 5a90e24aa8a6..911437c163c7 100644 --- a/arch/sh/configs/cayman_defconfig +++ b/arch/sh/configs/cayman_defconfig @@ -19,7 +19,6 @@ CONFIG_UNIX=y CONFIG_INET=y CONFIG_IP_PNP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y diff --git a/arch/sh/configs/dreamcast_defconfig b/arch/sh/configs/dreamcast_defconfig index 1d27666c029f..ae067e0b15e3 100644 --- a/arch/sh/configs/dreamcast_defconfig +++ b/arch/sh/configs/dreamcast_defconfig @@ -32,7 +32,6 @@ CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set # CONFIG_FW_LOADER is not set CONFIG_GDROM=y diff --git a/arch/sh/configs/ecovec24-romimage_defconfig b/arch/sh/configs/ecovec24-romimage_defconfig index bdb61d1d0127..5c60e71d839e 100644 --- a/arch/sh/configs/ecovec24-romimage_defconfig +++ b/arch/sh/configs/ecovec24-romimage_defconfig @@ -26,7 +26,6 @@ CONFIG_INET=y # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y # CONFIG_SCSI_LOWLEVEL is not set diff --git a/arch/sh/configs/ecovec24_defconfig b/arch/sh/configs/ecovec24_defconfig index ba67e3752938..2fb7db4957ce 100644 --- a/arch/sh/configs/ecovec24_defconfig +++ b/arch/sh/configs/ecovec24_defconfig @@ -31,7 +31,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set CONFIG_IRDA=y CONFIG_SH_SIR=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/edosk7760_defconfig b/arch/sh/configs/edosk7760_defconfig index aab4ff1e247c..02ba62298576 100644 --- a/arch/sh/configs/edosk7760_defconfig +++ b/arch/sh/configs/edosk7760_defconfig @@ -31,7 +31,6 @@ CONFIG_IP_PNP_BOOTP=y # CONFIG_INET_XFRM_MODE_TUNNEL is not set # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_DEBUG_DRIVER=y CONFIG_DEBUG_DEVRES=y diff --git a/arch/sh/configs/espt_defconfig b/arch/sh/configs/espt_defconfig index 444d75947e70..a5b865a75d22 100644 --- a/arch/sh/configs/espt_defconfig +++ b/arch/sh/configs/espt_defconfig @@ -26,7 +26,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/hp6xx_defconfig b/arch/sh/configs/hp6xx_defconfig index 91d43e2bffea..a92db6694ce2 100644 --- a/arch/sh/configs/hp6xx_defconfig +++ b/arch/sh/configs/hp6xx_defconfig @@ -18,7 +18,6 @@ CONFIG_HD64461_ENABLER=y CONFIG_PCCARD=y CONFIG_PM=y CONFIG_APM_EMULATION=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set CONFIG_BLK_DEV_SD=y CONFIG_ATA=y diff --git a/arch/sh/configs/kfr2r09-romimage_defconfig b/arch/sh/configs/kfr2r09-romimage_defconfig index 9cc37f29e3b4..04436b4fbd76 100644 --- a/arch/sh/configs/kfr2r09-romimage_defconfig +++ b/arch/sh/configs/kfr2r09-romimage_defconfig @@ -28,7 +28,6 @@ CONFIG_INET=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_INPUT_MOUSEDEV is not set # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set diff --git a/arch/sh/configs/kfr2r09_defconfig b/arch/sh/configs/kfr2r09_defconfig index 46693d033644..1dc3f670c481 100644 --- a/arch/sh/configs/kfr2r09_defconfig +++ b/arch/sh/configs/kfr2r09_defconfig @@ -35,7 +35,6 @@ CONFIG_INET=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/landisk_defconfig b/arch/sh/configs/landisk_defconfig index 467f4d2d8e87..567af752b1bb 100644 --- a/arch/sh/configs/landisk_defconfig +++ b/arch/sh/configs/landisk_defconfig @@ -26,7 +26,6 @@ CONFIG_IP_PNP=y # CONFIG_IPV6 is not set CONFIG_NETFILTER=y CONFIG_ATALK=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_IDE=y diff --git a/arch/sh/configs/lboxre2_defconfig b/arch/sh/configs/lboxre2_defconfig index 9e3edfdf9b2e..10f6d371ce2c 100644 --- a/arch/sh/configs/lboxre2_defconfig +++ b/arch/sh/configs/lboxre2_defconfig @@ -29,7 +29,6 @@ CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_PNP=y # CONFIG_IPV6 is not set CONFIG_NETFILTER=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_SD=y diff --git a/arch/sh/configs/magicpanelr2_defconfig b/arch/sh/configs/magicpanelr2_defconfig index fb7415dbc102..664c4dee6e6a 100644 --- a/arch/sh/configs/magicpanelr2_defconfig +++ b/arch/sh/configs/magicpanelr2_defconfig @@ -35,7 +35,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_XFRM_MODE_TUNNEL is not set # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set CONFIG_MTD=y diff --git a/arch/sh/configs/microdev_defconfig b/arch/sh/configs/microdev_defconfig index c3f7d5899922..ed84d1303acf 100644 --- a/arch/sh/configs/microdev_defconfig +++ b/arch/sh/configs/microdev_defconfig @@ -19,7 +19,6 @@ CONFIG_NET=y CONFIG_INET=y CONFIG_IP_PNP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_RAM=y CONFIG_IDE=y diff --git a/arch/sh/configs/migor_defconfig b/arch/sh/configs/migor_defconfig index 121a75d65fb4..494a1675c226 100644 --- a/arch/sh/configs/migor_defconfig +++ b/arch/sh/configs/migor_defconfig @@ -26,7 +26,6 @@ CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y diff --git a/arch/sh/configs/polaris_defconfig b/arch/sh/configs/polaris_defconfig index 87641b7d6c4e..e3a1d3d2694a 100644 --- a/arch/sh/configs/polaris_defconfig +++ b/arch/sh/configs/polaris_defconfig @@ -37,7 +37,6 @@ CONFIG_IP_MULTICAST=y # CONFIG_INET_XFRM_MODE_TUNNEL is not set # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/r7780mp_defconfig b/arch/sh/configs/r7780mp_defconfig index 435bcd66c667..0a18f8011c55 100644 --- a/arch/sh/configs/r7780mp_defconfig +++ b/arch/sh/configs/r7780mp_defconfig @@ -36,7 +36,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set CONFIG_BRIDGE=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_MTD=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/r7785rp_defconfig b/arch/sh/configs/r7785rp_defconfig index 5877e6d1f285..7226ac5a1d44 100644 --- a/arch/sh/configs/r7785rp_defconfig +++ b/arch/sh/configs/r7785rp_defconfig @@ -43,7 +43,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set CONFIG_BRIDGE=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_SD=y diff --git a/arch/sh/configs/rsk7201_defconfig b/arch/sh/configs/rsk7201_defconfig index b195bc01e406..9f4f474705b7 100644 --- a/arch/sh/configs/rsk7201_defconfig +++ b/arch/sh/configs/rsk7201_defconfig @@ -31,7 +31,6 @@ CONFIG_BINFMT_ZFLAT=y CONFIG_BINFMT_SHARED_FLAT=y CONFIG_PM=y CONFIG_CPU_IDLE=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_FW_LOADER is not set diff --git a/arch/sh/configs/rsk7203_defconfig b/arch/sh/configs/rsk7203_defconfig index 8c471959bbc7..10a32bd4cf66 100644 --- a/arch/sh/configs/rsk7203_defconfig +++ b/arch/sh/configs/rsk7203_defconfig @@ -45,7 +45,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_FW_LOADER is not set diff --git a/arch/sh/configs/rsk7264_defconfig b/arch/sh/configs/rsk7264_defconfig index ad003ee469ea..2b0572b497c1 100644 --- a/arch/sh/configs/rsk7264_defconfig +++ b/arch/sh/configs/rsk7264_defconfig @@ -36,7 +36,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_LOOP=y diff --git a/arch/sh/configs/rsk7269_defconfig b/arch/sh/configs/rsk7269_defconfig index 27fc01d58cf8..fb9fa7faf635 100644 --- a/arch/sh/configs/rsk7269_defconfig +++ b/arch/sh/configs/rsk7269_defconfig @@ -25,7 +25,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y diff --git a/arch/sh/configs/rts7751r2d1_defconfig b/arch/sh/configs/rts7751r2d1_defconfig index 379d673f5ce8..6a3cfe08295f 100644 --- a/arch/sh/configs/rts7751r2d1_defconfig +++ b/arch/sh/configs/rts7751r2d1_defconfig @@ -22,7 +22,6 @@ CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_SD=y diff --git a/arch/sh/configs/rts7751r2dplus_defconfig b/arch/sh/configs/rts7751r2dplus_defconfig index 11177bceda83..2b3d7d280672 100644 --- a/arch/sh/configs/rts7751r2dplus_defconfig +++ b/arch/sh/configs/rts7751r2dplus_defconfig @@ -22,7 +22,6 @@ CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y diff --git a/arch/sh/configs/sdk7780_defconfig b/arch/sh/configs/sdk7780_defconfig index 95e5208b8260..d10a0414123a 100644 --- a/arch/sh/configs/sdk7780_defconfig +++ b/arch/sh/configs/sdk7780_defconfig @@ -41,7 +41,6 @@ CONFIG_IP_PNP_BOOTP=y CONFIG_IPV6=y # CONFIG_INET6_XFRM_MODE_BEET is not set CONFIG_NET_SCHED=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_PARPORT=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y diff --git a/arch/sh/configs/sdk7786_defconfig b/arch/sh/configs/sdk7786_defconfig index 49a29338789b..7fa116b436c3 100644 --- a/arch/sh/configs/sdk7786_defconfig +++ b/arch/sh/configs/sdk7786_defconfig @@ -91,7 +91,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y diff --git a/arch/sh/configs/se7206_defconfig b/arch/sh/configs/se7206_defconfig index 3553acd5edb1..a93402b3a319 100644 --- a/arch/sh/configs/se7206_defconfig +++ b/arch/sh/configs/se7206_defconfig @@ -58,7 +58,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_FW_LOADER is not set diff --git a/arch/sh/configs/se7343_defconfig b/arch/sh/configs/se7343_defconfig index fc77a67b16e7..06d067c842cd 100644 --- a/arch/sh/configs/se7343_defconfig +++ b/arch/sh/configs/se7343_defconfig @@ -28,7 +28,6 @@ CONFIG_INET=y CONFIG_SYN_COOKIES=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/se7712_defconfig b/arch/sh/configs/se7712_defconfig index 1e116529735f..9a527f978106 100644 --- a/arch/sh/configs/se7712_defconfig +++ b/arch/sh/configs/se7712_defconfig @@ -63,7 +63,6 @@ CONFIG_NET_SCH_NETEM=y CONFIG_NET_CLS_TCINDEX=y CONFIG_NET_CLS_ROUTE4=y CONFIG_NET_CLS_FW=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/se7721_defconfig b/arch/sh/configs/se7721_defconfig index c66e512719ab..3b0e1eb6e874 100644 --- a/arch/sh/configs/se7721_defconfig +++ b/arch/sh/configs/se7721_defconfig @@ -62,7 +62,6 @@ CONFIG_NET_SCH_NETEM=y CONFIG_NET_CLS_TCINDEX=y CONFIG_NET_CLS_ROUTE4=y CONFIG_NET_CLS_FW=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/se7722_defconfig b/arch/sh/configs/se7722_defconfig index ccc7fc423fde..88bf9e849008 100644 --- a/arch/sh/configs/se7722_defconfig +++ b/arch/sh/configs/se7722_defconfig @@ -26,7 +26,6 @@ CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_SD=y diff --git a/arch/sh/configs/se7724_defconfig b/arch/sh/configs/se7724_defconfig index 9f6d46d58554..0e8d5cc1e107 100644 --- a/arch/sh/configs/se7724_defconfig +++ b/arch/sh/configs/se7724_defconfig @@ -30,7 +30,6 @@ CONFIG_IP_PNP_DHCP=y # CONFIG_INET_XFRM_MODE_TUNNEL is not set # CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/sh03_defconfig b/arch/sh/configs/sh03_defconfig index 489ffdfb1517..e5beb625ab88 100644 --- a/arch/sh/configs/sh03_defconfig +++ b/arch/sh/configs/sh03_defconfig @@ -34,7 +34,6 @@ CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_FW_LOADER is not set diff --git a/arch/sh/configs/sh2007_defconfig b/arch/sh/configs/sh2007_defconfig index cbd6742eb423..99975db461d8 100644 --- a/arch/sh/configs/sh2007_defconfig +++ b/arch/sh/configs/sh2007_defconfig @@ -44,7 +44,6 @@ CONFIG_NET_IPIP=y # CONFIG_IPV6 is not set CONFIG_NETWORK_SECMARK=y CONFIG_NET_PKTGEN=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_CDROM_PKTCDVD=y diff --git a/arch/sh/configs/sh7710voipgw_defconfig b/arch/sh/configs/sh7710voipgw_defconfig index 65a1aad899c8..c86f28442a80 100644 --- a/arch/sh/configs/sh7710voipgw_defconfig +++ b/arch/sh/configs/sh7710voipgw_defconfig @@ -32,7 +32,6 @@ CONFIG_NET_CLS_BASIC=y CONFIG_NET_CLS_TCINDEX=y CONFIG_NET_CLS_ROUTE4=y CONFIG_NET_CLS_U32=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/sh7724_generic_defconfig b/arch/sh/configs/sh7724_generic_defconfig index d04bc27aa816..9adee9010319 100644 --- a/arch/sh/configs/sh7724_generic_defconfig +++ b/arch/sh/configs/sh7724_generic_defconfig @@ -15,7 +15,6 @@ CONFIG_KEXEC=y CONFIG_KEXEC_JUMP=y CONFIG_HIBERNATION=y CONFIG_CPU_IDLE=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_INPUT is not set # CONFIG_SERIO is not set diff --git a/arch/sh/configs/sh7757lcr_defconfig b/arch/sh/configs/sh7757lcr_defconfig index b0c4bc830fb8..9f2aed0b3bca 100644 --- a/arch/sh/configs/sh7757lcr_defconfig +++ b/arch/sh/configs/sh7757lcr_defconfig @@ -33,7 +33,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_IPV6=y # CONFIG_WIRELESS is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_MTD=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/sh7763rdp_defconfig b/arch/sh/configs/sh7763rdp_defconfig index 405bf62d22d0..d0a0aa74cecf 100644 --- a/arch/sh/configs/sh7763rdp_defconfig +++ b/arch/sh/configs/sh7763rdp_defconfig @@ -26,7 +26,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLKDEVS=y diff --git a/arch/sh/configs/sh7770_generic_defconfig b/arch/sh/configs/sh7770_generic_defconfig index e5b733c2d988..c17590f0df67 100644 --- a/arch/sh/configs/sh7770_generic_defconfig +++ b/arch/sh/configs/sh7770_generic_defconfig @@ -17,7 +17,6 @@ CONFIG_KEXEC_JUMP=y CONFIG_PM=y CONFIG_HIBERNATION=y CONFIG_CPU_IDLE=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_PREVENT_FIRMWARE_BUILD is not set # CONFIG_INPUT is not set # CONFIG_SERIO is not set diff --git a/arch/sh/configs/sh7785lcr_32bit_defconfig b/arch/sh/configs/sh7785lcr_32bit_defconfig index a89ccc15af23..9b885c14c400 100644 --- a/arch/sh/configs/sh7785lcr_32bit_defconfig +++ b/arch/sh/configs/sh7785lcr_32bit_defconfig @@ -43,7 +43,6 @@ CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/sh7785lcr_defconfig b/arch/sh/configs/sh7785lcr_defconfig index 5201bb78c6f9..1b88929083f7 100644 --- a/arch/sh/configs/sh7785lcr_defconfig +++ b/arch/sh/configs/sh7785lcr_defconfig @@ -26,7 +26,6 @@ CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_MTD=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/configs/shx3_defconfig b/arch/sh/configs/shx3_defconfig index 755c4f73c718..dc2be2514b62 100644 --- a/arch/sh/configs/shx3_defconfig +++ b/arch/sh/configs/shx3_defconfig @@ -59,7 +59,6 @@ CONFIG_CAN=m CONFIG_CAN_RAW=m CONFIG_CAN_BCM=m CONFIG_CAN_VCAN=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_SD=y diff --git a/arch/sh/configs/titan_defconfig b/arch/sh/configs/titan_defconfig index 171ab05ce4fc..4ec961ace688 100644 --- a/arch/sh/configs/titan_defconfig +++ b/arch/sh/configs/titan_defconfig @@ -142,7 +142,6 @@ CONFIG_GACT_PROB=y CONFIG_NET_ACT_MIRRED=m CONFIG_NET_ACT_IPT=m CONFIG_NET_ACT_PEDIT=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_FW_LOADER=m CONFIG_CONNECTOR=m CONFIG_MTD=m diff --git a/arch/sh/configs/ul2_defconfig b/arch/sh/configs/ul2_defconfig index 1b7412df12e0..dc2e3061130f 100644 --- a/arch/sh/configs/ul2_defconfig +++ b/arch/sh/configs/ul2_defconfig @@ -33,7 +33,6 @@ CONFIG_CFG80211=y CONFIG_MAC80211=y CONFIG_MAC80211_RC_PID=y # CONFIG_MAC80211_RC_MINSTREL is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_MTD=y CONFIG_MTD_BLOCK=y CONFIG_MTD_CFI=y diff --git a/arch/sh/configs/urquell_defconfig b/arch/sh/configs/urquell_defconfig index f891045e633a..cb2f56468fe0 100644 --- a/arch/sh/configs/urquell_defconfig +++ b/arch/sh/configs/urquell_defconfig @@ -46,7 +46,6 @@ CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y # CONFIG_IPV6 is not set -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_FW_LOADER is not set CONFIG_MTD=y CONFIG_MTD_BLOCK=y diff --git a/arch/sh/kernel/kprobes.c b/arch/sh/kernel/kprobes.c index 1f8c0d30567f..318296f48f1a 100644 --- a/arch/sh/kernel/kprobes.c +++ b/arch/sh/kernel/kprobes.c @@ -485,7 +485,8 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); addr = (kprobe_opcode_t *) (args->regs->pc); - if (val == DIE_TRAP) { + if (val == DIE_TRAP && + args->trapnr == (BREAKPOINT_INSTRUCTION & 0xff)) { if (!kprobe_running()) { if (kprobe_handler(args->regs)) { ret = NOTIFY_STOP; diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 3093bc372138..5f51456f4fc7 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -24,20 +24,6 @@ #include <asm/tlbflush.h> #include <asm/traps.h> -static inline int notify_page_fault(struct pt_regs *regs, int trap) -{ - int ret = 0; - - if (kprobes_built_in() && !user_mode(regs)) { - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, trap)) - ret = 1; - preempt_enable(); - } - - return ret; -} - static void force_sig_info_fault(int si_signo, int si_code, unsigned long address) { @@ -412,14 +398,14 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, if (unlikely(fault_in_kernel_space(address))) { if (vmalloc_fault(address) >= 0) return; - if (notify_page_fault(regs, vec)) + if (kprobe_page_fault(regs, vec)) return; bad_area_nosemaphore(regs, error_code, address); return; } - if (unlikely(notify_page_fault(regs, vec))) + if (unlikely(kprobe_page_fault(regs, vec))) return; /* Only enable interrupts if they were on before the fault */ diff --git a/arch/sparc/include/uapi/asm/mman.h b/arch/sparc/include/uapi/asm/mman.h index f6f99ec65bb3..cec9f4109687 100644 --- a/arch/sparc/include/uapi/asm/mman.h +++ b/arch/sparc/include/uapi/asm/mman.h @@ -22,10 +22,4 @@ #define MCL_FUTURE 0x4000 /* lock all additions to address space */ #define MCL_ONFAULT 0x8000 /* lock all pages that are faulted in */ -#define MAP_POPULATE 0x8000 /* populate (prefault) pagetables */ -#define MAP_NONBLOCK 0x10000 /* do not block on IO */ -#define MAP_STACK 0x20000 /* give out an address that is best suited for process/thread stacks */ -#define MAP_HUGETLB 0x40000 /* create a huge page mapping */ - - #endif /* _UAPI__SPARC_MMAN_H__ */ diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 83fda4d9c3b2..2371fb6b97e4 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -38,20 +38,6 @@ int show_unhandled_signals = 1; -static inline __kprobes int notify_page_fault(struct pt_regs *regs) -{ - int ret = 0; - - /* kprobe_running() needs smp_processor_id() */ - if (kprobes_built_in() && !user_mode(regs)) { - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, 0)) - ret = 1; - preempt_enable(); - } - return ret; -} - static void __kprobes unhandled_fault(unsigned long address, struct task_struct *tsk, struct pt_regs *regs) @@ -285,7 +271,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) fault_code = get_thread_fault_code(); - if (notify_page_fault(regs)) + if (kprobe_page_fault(regs, 0)) goto exit_exception; si_code = SEGV_MAPERR; diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 57d3b5d96bd2..1342654e8057 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -70,6 +70,7 @@ config X86 select ARCH_HAS_KCOV if X86_64 select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_PMEM_API if X86_64 + select ARCH_HAS_PTE_DEVMAP if X86_64 select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_REFCOUNT select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 @@ -80,7 +81,6 @@ config X86 select ARCH_HAS_STRICT_MODULE_RWX select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_HAS_ZONE_DEVICE if X86_64 select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI select ARCH_MIGHT_HAVE_PC_PARPORT diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index a06a9f8294ea..6bed97ff6db2 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -165,7 +165,6 @@ static inline unsigned int isa_virt_to_bus(volatile void *address) { return (unsigned int)virt_to_phys(address); } -#define isa_page_to_bus(page) ((unsigned int)page_to_phys(page)) #define isa_bus_to_virt phys_to_virt /* diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 5e0509b41986..0bc530c4eb13 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -271,7 +271,7 @@ static inline int has_transparent_hugepage(void) return boot_cpu_has(X86_FEATURE_PSE); } -#ifdef __HAVE_ARCH_PTE_DEVMAP +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP static inline int pmd_devmap(pmd_t pmd) { return !!(pmd_val(pmd) & _PAGE_DEVMAP); @@ -732,7 +732,7 @@ static inline int pte_present(pte_t a) return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE); } -#ifdef __HAVE_ARCH_PTE_DEVMAP +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP static inline int pte_devmap(pte_t a) { return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index d6ff0bbdb394..b5e49e6bac63 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -103,7 +103,6 @@ #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) #define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) #define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) -#define __HAVE_ARCH_PTE_DEVMAP #else #define _PAGE_NX (_AT(pteval_t, 0)) #define _PAGE_DEVMAP (_AT(pteval_t, 0)) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index c82abd6e4ca3..9c4435307ff8 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -66,7 +66,9 @@ static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, un }) #ifdef CONFIG_DEBUG_ATOMIC_SLEEP -# define WARN_ON_IN_IRQ() WARN_ON_ONCE(!in_task()) +static inline bool pagefault_disabled(void); +# define WARN_ON_IN_IRQ() \ + WARN_ON_ONCE(!in_task() && !pagefault_disabled()) #else # define WARN_ON_IN_IRQ() #endif diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 4b73f5937f41..024c3053dbba 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -373,7 +373,7 @@ static int add_brk_on_nop(struct dyn_ftrace *rec) return add_break(rec->ip, old); } -static int add_breakpoints(struct dyn_ftrace *rec, int enable) +static int add_breakpoints(struct dyn_ftrace *rec, bool enable) { unsigned long ftrace_addr; int ret; @@ -481,7 +481,7 @@ static int add_update_nop(struct dyn_ftrace *rec) return add_update_code(ip, new); } -static int add_update(struct dyn_ftrace *rec, int enable) +static int add_update(struct dyn_ftrace *rec, bool enable) { unsigned long ftrace_addr; int ret; @@ -527,7 +527,7 @@ static int finish_update_nop(struct dyn_ftrace *rec) return ftrace_write(ip, new, 1); } -static int finish_update(struct dyn_ftrace *rec, int enable) +static int finish_update(struct dyn_ftrace *rec, bool enable) { unsigned long ftrace_addr; int ret; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 794f364cb882..d1634c59ed56 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -46,23 +46,6 @@ kmmio_fault(struct pt_regs *regs, unsigned long addr) return 0; } -static nokprobe_inline int kprobes_fault(struct pt_regs *regs) -{ - if (!kprobes_built_in()) - return 0; - if (user_mode(regs)) - return 0; - /* - * To be potentially processing a kprobe fault and to be allowed to call - * kprobe_running(), we have to be non-preemptible. - */ - if (preemptible()) - return 0; - if (!kprobe_running()) - return 0; - return kprobe_fault_handler(regs, X86_TRAP_PF); -} - /* * Prefetch quirks: * @@ -1282,7 +1265,7 @@ do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code, return; /* kprobes don't want to hook the spurious faults: */ - if (kprobes_fault(regs)) + if (kprobe_page_fault(regs, X86_TRAP_PF)) return; /* @@ -1313,7 +1296,7 @@ void do_user_addr_fault(struct pt_regs *regs, mm = tsk->mm; /* kprobes don't want to hook the spurious faults: */ - if (unlikely(kprobes_fault(regs))) + if (unlikely(kprobe_page_fault(regs, X86_TRAP_PF))) return; /* diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index e500f1df1140..63e99f15d7cf 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c @@ -459,6 +459,11 @@ void iounmap(volatile void __iomem *addr) } EXPORT_SYMBOL(iounmap); +int __init arch_ioremap_p4d_supported(void) +{ + return 0; +} + int __init arch_ioremap_pud_supported(void) { #ifdef CONFIG_X86_64 diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h index be726062412b..ebbb48842190 100644 --- a/arch/xtensa/include/uapi/asm/mman.h +++ b/arch/xtensa/include/uapi/asm/mman.h @@ -56,12 +56,8 @@ #define MAP_STACK 0x40000 /* give out an address that is best suited for process/thread stacks */ #define MAP_HUGETLB 0x80000 /* create a huge page mapping */ #define MAP_FIXED_NOREPLACE 0x100000 /* MAP_FIXED which doesn't unmap underlying mapping */ -#ifdef CONFIG_MMAP_ALLOW_UNINITIALIZED -# define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be +#define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be * uninitialized */ -#else -# define MAP_UNINITIALIZED 0x0 /* Don't support this flag */ -#endif /* * Flags for msync diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index 9489ffc06411..4f325e47519f 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -60,6 +60,12 @@ module_param(report_key_events, int, 0644); MODULE_PARM_DESC(report_key_events, "0: none, 1: output changes, 2: brightness changes, 3: all"); +static int hw_changes_brightness = -1; +module_param(hw_changes_brightness, int, 0644); +MODULE_PARM_DESC(hw_changes_brightness, + "Set this to 1 on buggy hw which changes the brightness itself when " + "a hotkey is pressed: -1: auto, 0: normal 1: hw-changes-brightness"); + /* * Whether the struct acpi_video_device_attrib::device_id_scheme bit should be * assumed even if not actually set. @@ -405,6 +411,14 @@ static int video_set_report_key_events(const struct dmi_system_id *id) return 0; } +static int video_hw_changes_brightness( + const struct dmi_system_id *d) +{ + if (hw_changes_brightness == -1) + hw_changes_brightness = 1; + return 0; +} + static const struct dmi_system_id video_dmi_table[] = { /* * Broken _BQC workaround http://bugzilla.kernel.org/show_bug.cgi?id=13121 @@ -529,6 +543,21 @@ static const struct dmi_system_id video_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Vostro V131"), }, }, + /* + * Some machines change the brightness themselves when a brightness + * hotkey gets pressed, despite us telling them not to. In this case + * acpi_video_device_notify() should only call backlight_force_update( + * BACKLIGHT_UPDATE_HOTKEY) and not do anything else. + */ + { + /* https://bugzilla.kernel.org/show_bug.cgi?id=204077 */ + .callback = video_hw_changes_brightness, + .ident = "Packard Bell EasyNote MZ35", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Packard Bell"), + DMI_MATCH(DMI_PRODUCT_NAME, "EasyNote MZ35"), + }, + }, {} }; @@ -1612,6 +1641,14 @@ static void acpi_video_device_notify(acpi_handle handle, u32 event, void *data) bus = video_device->video; input = bus->input; + if (hw_changes_brightness > 0) { + if (video_device->backlight) + backlight_force_update(video_device->backlight, + BACKLIGHT_UPDATE_HOTKEY); + acpi_notifier_call_chain(device, event, 0); + return; + } + switch (event) { case ACPI_VIDEO_NOTIFY_CYCLE_BRIGHTNESS: /* Cycle brightness */ brightness_switch_event(video_device, event); diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c index 587aeeeb5070..46a8baf28bd0 100644 --- a/drivers/acpi/acpica/exconfig.c +++ b/drivers/acpi/acpica/exconfig.c @@ -174,12 +174,11 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state, return_ACPI_STATUS(status); } - /* Complete the initialization/resolution of package objects */ + /* Complete the initialization/resolution of new objects */ - status = acpi_ns_walk_namespace(ACPI_TYPE_PACKAGE, ACPI_ROOT_OBJECT, - ACPI_UINT32_MAX, 0, - acpi_ns_init_one_package, NULL, NULL, - NULL); + acpi_ex_exit_interpreter(); + acpi_ns_initialize_objects(); + acpi_ex_enter_interpreter(); /* Parameter Data (optional) */ @@ -437,12 +436,11 @@ acpi_ex_load_op(union acpi_operand_object *obj_desc, return_ACPI_STATUS(status); } - /* Complete the initialization/resolution of package objects */ + /* Complete the initialization/resolution of new objects */ - status = acpi_ns_walk_namespace(ACPI_TYPE_PACKAGE, ACPI_ROOT_OBJECT, - ACPI_UINT32_MAX, 0, - acpi_ns_init_one_package, NULL, NULL, - NULL); + acpi_ex_exit_interpreter(); + acpi_ns_initialize_objects(); + acpi_ex_enter_interpreter(); /* Store the ddb_handle into the Target operand */ diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c index ef8f8a9f3c9c..86f1693f6d29 100644 --- a/drivers/acpi/acpica/tbxfload.c +++ b/drivers/acpi/acpica/tbxfload.c @@ -297,15 +297,11 @@ acpi_status acpi_load_table(struct acpi_table_header *table) status = acpi_tb_install_and_load_table(ACPI_PTR_TO_PHYSADDR(table), ACPI_TABLE_ORIGIN_EXTERNAL_VIRTUAL, FALSE, &table_index); - if (ACPI_SUCCESS(status)) { - /* Complete the initialization/resolution of package objects */ - status = acpi_ns_walk_namespace(ACPI_TYPE_PACKAGE, - ACPI_ROOT_OBJECT, - ACPI_UINT32_MAX, 0, - acpi_ns_init_one_package, - NULL, NULL, NULL); + /* Complete the initialization/resolution of new objects */ + + acpi_ns_initialize_objects(); } return_ACPI_STATUS(status); diff --git a/drivers/acpi/blacklist.c b/drivers/acpi/blacklist.c index ad2c565f5cbe..a86a770c9b79 100644 --- a/drivers/acpi/blacklist.c +++ b/drivers/acpi/blacklist.c @@ -17,7 +17,9 @@ #include "internal.h" +#ifdef CONFIG_DMI static const struct dmi_system_id acpi_rev_dmi_table[] __initconst; +#endif /* * POLICY: If *anything* doesn't work, put it on the blacklist. @@ -61,7 +63,9 @@ int __init acpi_blacklisted(void) } (void)early_acpi_osi_init(); +#ifdef CONFIG_DMI dmi_check_system(acpi_rev_dmi_table); +#endif return blacklisted; } diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index 23022cf20d26..c02fa27dd3f3 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -2426,7 +2426,7 @@ static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, offset = to_interleave_offset(offset, mmio); writeq(cmd, mmio->addr.base + offset); - nvdimm_flush(nfit_blk->nd_region); + nvdimm_flush(nfit_blk->nd_region, NULL); if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) readq(mmio->addr.base + offset); @@ -2475,7 +2475,7 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, } if (rw) - nvdimm_flush(nfit_blk->nd_region); + nvdimm_flush(nfit_blk->nd_region, NULL); rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; return rc; diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 33c30c1e6a30..b063bc41b0a9 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -1536,7 +1536,8 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, if (ret) genpd_free_dev_data(dev, gpd_data); else - dev_pm_qos_add_notifier(dev, &gpd_data->nb); + dev_pm_qos_add_notifier(dev, &gpd_data->nb, + DEV_PM_QOS_RESUME_LATENCY); return ret; } @@ -1569,7 +1570,8 @@ static int genpd_remove_device(struct generic_pm_domain *genpd, pdd = dev->power.subsys_data->domain_data; gpd_data = to_gpd_data(pdd); - dev_pm_qos_remove_notifier(dev, &gpd_data->nb); + dev_pm_qos_remove_notifier(dev, &gpd_data->nb, + DEV_PM_QOS_RESUME_LATENCY); genpd_lock(genpd); @@ -1597,7 +1599,7 @@ static int genpd_remove_device(struct generic_pm_domain *genpd, out: genpd_unlock(genpd); - dev_pm_qos_add_notifier(dev, &gpd_data->nb); + dev_pm_qos_add_notifier(dev, &gpd_data->nb, DEV_PM_QOS_RESUME_LATENCY); return ret; } diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c index 3838045c9277..daa8c7689f7e 100644 --- a/drivers/base/power/domain_governor.c +++ b/drivers/base/power/domain_governor.c @@ -33,7 +33,7 @@ static int dev_update_qos_constraint(struct device *dev, void *data) * take its current PM QoS constraint (that's the only thing * known at this point anyway). */ - constraint_ns = dev_pm_qos_read_value(dev); + constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY); constraint_ns *= NSEC_PER_USEC; } @@ -66,7 +66,7 @@ static bool default_suspend_ok(struct device *dev) td->constraint_changed = false; td->cached_suspend_ok = false; td->effective_constraint_ns = 0; - constraint_ns = __dev_pm_qos_read_value(dev); + constraint_ns = __dev_pm_qos_resume_latency(dev); spin_unlock_irqrestore(&dev->power.lock, flags); diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index 6c91f8df1d59..6c90fd7e2ff8 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -90,29 +90,49 @@ enum pm_qos_flags_status dev_pm_qos_flags(struct device *dev, s32 mask) EXPORT_SYMBOL_GPL(dev_pm_qos_flags); /** - * __dev_pm_qos_read_value - Get PM QoS constraint for a given device. + * __dev_pm_qos_resume_latency - Get resume latency constraint for a given device. * @dev: Device to get the PM QoS constraint value for. * * This routine must be called with dev->power.lock held. */ -s32 __dev_pm_qos_read_value(struct device *dev) +s32 __dev_pm_qos_resume_latency(struct device *dev) { lockdep_assert_held(&dev->power.lock); - return dev_pm_qos_raw_read_value(dev); + return dev_pm_qos_raw_resume_latency(dev); } /** * dev_pm_qos_read_value - Get PM QoS constraint for a given device (locked). * @dev: Device to get the PM QoS constraint value for. + * @type: QoS request type. */ -s32 dev_pm_qos_read_value(struct device *dev) +s32 dev_pm_qos_read_value(struct device *dev, enum dev_pm_qos_req_type type) { + struct dev_pm_qos *qos = dev->power.qos; unsigned long flags; s32 ret; spin_lock_irqsave(&dev->power.lock, flags); - ret = __dev_pm_qos_read_value(dev); + + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: + ret = IS_ERR_OR_NULL(qos) ? PM_QOS_RESUME_LATENCY_NO_CONSTRAINT + : pm_qos_read_value(&qos->resume_latency); + break; + case DEV_PM_QOS_MIN_FREQUENCY: + ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE + : pm_qos_read_value(&qos->min_frequency); + break; + case DEV_PM_QOS_MAX_FREQUENCY: + ret = IS_ERR_OR_NULL(qos) ? PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE + : pm_qos_read_value(&qos->max_frequency); + break; + default: + WARN_ON(1); + ret = 0; + } + spin_unlock_irqrestore(&dev->power.lock, flags); return ret; @@ -149,6 +169,14 @@ static int apply_constraint(struct dev_pm_qos_request *req, req->dev->power.set_latency_tolerance(req->dev, value); } break; + case DEV_PM_QOS_MIN_FREQUENCY: + ret = pm_qos_update_target(&qos->min_frequency, + &req->data.pnode, action, value); + break; + case DEV_PM_QOS_MAX_FREQUENCY: + ret = pm_qos_update_target(&qos->max_frequency, + &req->data.pnode, action, value); + break; case DEV_PM_QOS_FLAGS: ret = pm_qos_update_flags(&qos->flags, &req->data.flr, action, value); @@ -177,12 +205,11 @@ static int dev_pm_qos_constraints_allocate(struct device *dev) if (!qos) return -ENOMEM; - n = kzalloc(sizeof(*n), GFP_KERNEL); + n = kzalloc(3 * sizeof(*n), GFP_KERNEL); if (!n) { kfree(qos); return -ENOMEM; } - BLOCKING_INIT_NOTIFIER_HEAD(n); c = &qos->resume_latency; plist_head_init(&c->list); @@ -191,6 +218,7 @@ static int dev_pm_qos_constraints_allocate(struct device *dev) c->no_constraint_value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; c->type = PM_QOS_MIN; c->notifiers = n; + BLOCKING_INIT_NOTIFIER_HEAD(n); c = &qos->latency_tolerance; plist_head_init(&c->list); @@ -199,6 +227,24 @@ static int dev_pm_qos_constraints_allocate(struct device *dev) c->no_constraint_value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT; c->type = PM_QOS_MIN; + c = &qos->min_frequency; + plist_head_init(&c->list); + c->target_value = PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE; + c->default_value = PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE; + c->no_constraint_value = PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE; + c->type = PM_QOS_MAX; + c->notifiers = ++n; + BLOCKING_INIT_NOTIFIER_HEAD(n); + + c = &qos->max_frequency; + plist_head_init(&c->list); + c->target_value = PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE; + c->default_value = PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE; + c->no_constraint_value = PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE; + c->type = PM_QOS_MIN; + c->notifiers = ++n; + BLOCKING_INIT_NOTIFIER_HEAD(n); + INIT_LIST_HEAD(&qos->flags.list); spin_lock_irq(&dev->power.lock); @@ -252,11 +298,25 @@ void dev_pm_qos_constraints_destroy(struct device *dev) apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); memset(req, 0, sizeof(*req)); } + c = &qos->latency_tolerance; plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); memset(req, 0, sizeof(*req)); } + + c = &qos->min_frequency; + plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { + apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE); + memset(req, 0, sizeof(*req)); + } + + c = &qos->max_frequency; + plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { + apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); + memset(req, 0, sizeof(*req)); + } + f = &qos->flags; list_for_each_entry_safe(req, tmp, &f->list, data.flr.node) { apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); @@ -368,6 +428,8 @@ static int __dev_pm_qos_update_request(struct dev_pm_qos_request *req, switch(req->type) { case DEV_PM_QOS_RESUME_LATENCY: case DEV_PM_QOS_LATENCY_TOLERANCE: + case DEV_PM_QOS_MIN_FREQUENCY: + case DEV_PM_QOS_MAX_FREQUENCY: curr_value = req->data.pnode.prio; break; case DEV_PM_QOS_FLAGS: @@ -467,6 +529,7 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_remove_request); * * @dev: target device for the constraint * @notifier: notifier block managed by caller. + * @type: request type. * * Will register the notifier into a notification chain that gets called * upon changes to the target value for the device. @@ -474,7 +537,8 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_remove_request); * If the device's constraints object doesn't exist when this routine is called, * it will be created (or error code will be returned if that fails). */ -int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier) +int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier, + enum dev_pm_qos_req_type type) { int ret = 0; @@ -485,10 +549,28 @@ int dev_pm_qos_add_notifier(struct device *dev, struct notifier_block *notifier) else if (!dev->power.qos) ret = dev_pm_qos_constraints_allocate(dev); - if (!ret) + if (ret) + goto unlock; + + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: ret = blocking_notifier_chain_register(dev->power.qos->resume_latency.notifiers, notifier); + break; + case DEV_PM_QOS_MIN_FREQUENCY: + ret = blocking_notifier_chain_register(dev->power.qos->min_frequency.notifiers, + notifier); + break; + case DEV_PM_QOS_MAX_FREQUENCY: + ret = blocking_notifier_chain_register(dev->power.qos->max_frequency.notifiers, + notifier); + break; + default: + WARN_ON(1); + ret = -EINVAL; + } +unlock: mutex_unlock(&dev_pm_qos_mtx); return ret; } @@ -500,24 +582,44 @@ EXPORT_SYMBOL_GPL(dev_pm_qos_add_notifier); * * @dev: target device for the constraint * @notifier: notifier block to be removed. + * @type: request type. * * Will remove the notifier from the notification chain that gets called * upon changes to the target value. */ int dev_pm_qos_remove_notifier(struct device *dev, - struct notifier_block *notifier) + struct notifier_block *notifier, + enum dev_pm_qos_req_type type) { - int retval = 0; + int ret = 0; mutex_lock(&dev_pm_qos_mtx); /* Silently return if the constraints object is not present. */ - if (!IS_ERR_OR_NULL(dev->power.qos)) - retval = blocking_notifier_chain_unregister(dev->power.qos->resume_latency.notifiers, - notifier); + if (IS_ERR_OR_NULL(dev->power.qos)) + goto unlock; + + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: + ret = blocking_notifier_chain_unregister(dev->power.qos->resume_latency.notifiers, + notifier); + break; + case DEV_PM_QOS_MIN_FREQUENCY: + ret = blocking_notifier_chain_unregister(dev->power.qos->min_frequency.notifiers, + notifier); + break; + case DEV_PM_QOS_MAX_FREQUENCY: + ret = blocking_notifier_chain_unregister(dev->power.qos->max_frequency.notifiers, + notifier); + break; + default: + WARN_ON(1); + ret = -EINVAL; + } +unlock: mutex_unlock(&dev_pm_qos_mtx); - return retval; + return ret; } EXPORT_SYMBOL_GPL(dev_pm_qos_remove_notifier); @@ -577,6 +679,9 @@ static void __dev_pm_qos_drop_user_request(struct device *dev, req = dev->power.qos->flags_req; dev->power.qos->flags_req = NULL; break; + default: + WARN_ON(1); + return; } __dev_pm_qos_remove_request(req); kfree(req); diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 952a1e7057c7..b75335508d2c 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -275,7 +275,7 @@ static int rpm_check_suspend_allowed(struct device *dev) || (dev->power.request_pending && dev->power.request == RPM_REQ_RESUME)) retval = -EAGAIN; - else if (__dev_pm_qos_read_value(dev) == 0) + else if (__dev_pm_qos_resume_latency(dev) == 0) retval = -EPERM; else if (dev->power.runtime_status == RPM_SUSPENDED) retval = 1; diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index f652c1ac3ae9..0469aceaa230 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -2120,6 +2120,9 @@ static void setup_format_params(int track) raw_cmd->kernel_data = floppy_track_buffer; raw_cmd->length = 4 * F_SECT_PER_TRACK; + if (!F_SECT_PER_TRACK) + return; + /* allow for about 30ms for data transport per track */ head_shift = (F_SECT_PER_TRACK + 5) / 6; @@ -3230,8 +3233,12 @@ static int set_geometry(unsigned int cmd, struct floppy_struct *g, int cnt; /* sanity checking for parameters. */ - if (g->sect <= 0 || - g->head <= 0 || + if ((int)g->sect <= 0 || + (int)g->head <= 0 || + /* check for overflow in max_sector */ + (int)(g->sect * g->head) <= 0 || + /* check for zero in F_SECT_PER_TRACK */ + (unsigned char)((g->sect << 2) >> FD_SIZECODE(g)) == 0 || g->track <= 0 || g->track > UDP->tracks >> STRETCH(g) || /* check if reserved bits are set */ (g->stretch & ~(FD_STRETCH | FD_SWAPSIDES | FD_SECTBASEMASK)) != 0) @@ -3375,6 +3382,24 @@ static int fd_getgeo(struct block_device *bdev, struct hd_geometry *geo) return 0; } +static bool valid_floppy_drive_params(const short autodetect[8], + int native_format) +{ + size_t floppy_type_size = ARRAY_SIZE(floppy_type); + size_t i = 0; + + for (i = 0; i < 8; ++i) { + if (autodetect[i] < 0 || + autodetect[i] >= floppy_type_size) + return false; + } + + if (native_format < 0 || native_format >= floppy_type_size) + return false; + + return true; +} + static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long param) { @@ -3501,6 +3526,9 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int SUPBOUND(size, strlen((const char *)outparam) + 1); break; case FDSETDRVPRM: + if (!valid_floppy_drive_params(inparam.dp.autodetect, + inparam.dp.native_format)) + return -EINVAL; *UDP = inparam.dp; break; case FDGETDRVPRM: @@ -3698,6 +3726,8 @@ static int compat_setdrvprm(int drive, return -EPERM; if (copy_from_user(&v, arg, sizeof(struct compat_floppy_drive_params))) return -EFAULT; + if (!valid_floppy_drive_params(v.autodetect, v.native_format)) + return -EINVAL; mutex_lock(&floppy_mutex); UDP->cmos = v.cmos; UDP->max_dtr = v.max_dtr; diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index e5009a34f9c2..3327192bb71f 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -115,6 +115,8 @@ static int atomic_dec_return_safe(atomic_t *v) #define RBD_FEATURE_LAYERING (1ULL<<0) #define RBD_FEATURE_STRIPINGV2 (1ULL<<1) #define RBD_FEATURE_EXCLUSIVE_LOCK (1ULL<<2) +#define RBD_FEATURE_OBJECT_MAP (1ULL<<3) +#define RBD_FEATURE_FAST_DIFF (1ULL<<4) #define RBD_FEATURE_DEEP_FLATTEN (1ULL<<5) #define RBD_FEATURE_DATA_POOL (1ULL<<7) #define RBD_FEATURE_OPERATIONS (1ULL<<8) @@ -122,6 +124,8 @@ static int atomic_dec_return_safe(atomic_t *v) #define RBD_FEATURES_ALL (RBD_FEATURE_LAYERING | \ RBD_FEATURE_STRIPINGV2 | \ RBD_FEATURE_EXCLUSIVE_LOCK | \ + RBD_FEATURE_OBJECT_MAP | \ + RBD_FEATURE_FAST_DIFF | \ RBD_FEATURE_DEEP_FLATTEN | \ RBD_FEATURE_DATA_POOL | \ RBD_FEATURE_OPERATIONS) @@ -203,6 +207,11 @@ struct rbd_client { struct list_head node; }; +struct pending_result { + int result; /* first nonzero result */ + int num_pending; +}; + struct rbd_img_request; enum obj_request_type { @@ -219,6 +228,18 @@ enum obj_operation_type { OBJ_OP_ZEROOUT, }; +#define RBD_OBJ_FLAG_DELETION (1U << 0) +#define RBD_OBJ_FLAG_COPYUP_ENABLED (1U << 1) +#define RBD_OBJ_FLAG_COPYUP_ZEROS (1U << 2) +#define RBD_OBJ_FLAG_MAY_EXIST (1U << 3) +#define RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT (1U << 4) + +enum rbd_obj_read_state { + RBD_OBJ_READ_START = 1, + RBD_OBJ_READ_OBJECT, + RBD_OBJ_READ_PARENT, +}; + /* * Writes go through the following state machine to deal with * layering: @@ -245,17 +266,28 @@ enum obj_operation_type { * even if there is a parent). */ enum rbd_obj_write_state { - RBD_OBJ_WRITE_FLAT = 1, - RBD_OBJ_WRITE_GUARD, - RBD_OBJ_WRITE_READ_FROM_PARENT, - RBD_OBJ_WRITE_COPYUP_EMPTY_SNAPC, - RBD_OBJ_WRITE_COPYUP_OPS, + RBD_OBJ_WRITE_START = 1, + RBD_OBJ_WRITE_PRE_OBJECT_MAP, + RBD_OBJ_WRITE_OBJECT, + __RBD_OBJ_WRITE_COPYUP, + RBD_OBJ_WRITE_COPYUP, + RBD_OBJ_WRITE_POST_OBJECT_MAP, +}; + +enum rbd_obj_copyup_state { + RBD_OBJ_COPYUP_START = 1, + RBD_OBJ_COPYUP_READ_PARENT, + __RBD_OBJ_COPYUP_OBJECT_MAPS, + RBD_OBJ_COPYUP_OBJECT_MAPS, + __RBD_OBJ_COPYUP_WRITE_OBJECT, + RBD_OBJ_COPYUP_WRITE_OBJECT, }; struct rbd_obj_request { struct ceph_object_extent ex; + unsigned int flags; /* RBD_OBJ_FLAG_* */ union { - bool tried_parent; /* for reads */ + enum rbd_obj_read_state read_state; /* for reads */ enum rbd_obj_write_state write_state; /* for writes */ }; @@ -271,14 +303,15 @@ struct rbd_obj_request { u32 bvec_idx; }; }; + + enum rbd_obj_copyup_state copyup_state; struct bio_vec *copyup_bvecs; u32 copyup_bvec_count; - struct ceph_osd_request *osd_req; - - u64 xferred; /* bytes transferred */ - int result; + struct list_head osd_reqs; /* w/ r_private_item */ + struct mutex state_mutex; + struct pending_result pending; struct kref kref; }; @@ -287,11 +320,19 @@ enum img_req_flags { IMG_REQ_LAYERED, /* ENOENT handling: normal = 0, layered = 1 */ }; +enum rbd_img_state { + RBD_IMG_START = 1, + RBD_IMG_EXCLUSIVE_LOCK, + __RBD_IMG_OBJECT_REQUESTS, + RBD_IMG_OBJECT_REQUESTS, +}; + struct rbd_img_request { struct rbd_device *rbd_dev; enum obj_operation_type op_type; enum obj_request_type data_type; unsigned long flags; + enum rbd_img_state state; union { u64 snap_id; /* for reads */ struct ceph_snap_context *snapc; /* for writes */ @@ -300,13 +341,14 @@ struct rbd_img_request { struct request *rq; /* block request */ struct rbd_obj_request *obj_request; /* obj req initiator */ }; - spinlock_t completion_lock; - u64 xferred;/* aggregate bytes transferred */ - int result; /* first nonzero obj_request result */ + struct list_head lock_item; struct list_head object_extents; /* obj_req.ex structs */ - u32 pending_count; + struct mutex state_mutex; + struct pending_result pending; + struct work_struct work; + int work_result; struct kref kref; }; @@ -380,7 +422,17 @@ struct rbd_device { struct work_struct released_lock_work; struct delayed_work lock_dwork; struct work_struct unlock_work; - wait_queue_head_t lock_waitq; + spinlock_t lock_lists_lock; + struct list_head acquiring_list; + struct list_head running_list; + struct completion acquire_wait; + int acquire_err; + struct completion releasing_wait; + + spinlock_t object_map_lock; + u8 *object_map; + u64 object_map_size; /* in objects */ + u64 object_map_flags; struct workqueue_struct *task_wq; @@ -408,12 +460,10 @@ struct rbd_device { * Flag bits for rbd_dev->flags: * - REMOVING (which is coupled with rbd_dev->open_count) is protected * by rbd_dev->lock - * - BLACKLISTED is protected by rbd_dev->lock_rwsem */ enum rbd_dev_flags { RBD_DEV_FLAG_EXISTS, /* mapped snapshot has not been deleted */ RBD_DEV_FLAG_REMOVING, /* this mapping is being removed */ - RBD_DEV_FLAG_BLACKLISTED, /* our ceph_client is blacklisted */ }; static DEFINE_MUTEX(client_mutex); /* Serialize client creation */ @@ -466,6 +516,8 @@ static int minor_to_rbd_dev_id(int minor) static bool __rbd_is_lock_owner(struct rbd_device *rbd_dev) { + lockdep_assert_held(&rbd_dev->lock_rwsem); + return rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED || rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING; } @@ -583,6 +635,26 @@ static int _rbd_dev_v2_snap_size(struct rbd_device *rbd_dev, u64 snap_id, u8 *order, u64 *snap_size); static int _rbd_dev_v2_snap_features(struct rbd_device *rbd_dev, u64 snap_id, u64 *snap_features); +static int rbd_dev_v2_get_flags(struct rbd_device *rbd_dev); + +static void rbd_obj_handle_request(struct rbd_obj_request *obj_req, int result); +static void rbd_img_handle_request(struct rbd_img_request *img_req, int result); + +/* + * Return true if nothing else is pending. + */ +static bool pending_result_dec(struct pending_result *pending, int *result) +{ + rbd_assert(pending->num_pending > 0); + + if (*result && !pending->result) + pending->result = *result; + if (--pending->num_pending) + return false; + + *result = pending->result; + return true; +} static int rbd_open(struct block_device *bdev, fmode_t mode) { @@ -1317,6 +1389,8 @@ static void zero_bvecs(struct ceph_bvec_iter *bvec_pos, u32 off, u32 bytes) static void rbd_obj_zero_range(struct rbd_obj_request *obj_req, u32 off, u32 bytes) { + dout("%s %p data buf %u~%u\n", __func__, obj_req, off, bytes); + switch (obj_req->img_request->data_type) { case OBJ_REQUEST_BIO: zero_bios(&obj_req->bio_pos, off, bytes); @@ -1339,13 +1413,6 @@ static void rbd_obj_request_put(struct rbd_obj_request *obj_request) kref_put(&obj_request->kref, rbd_obj_request_destroy); } -static void rbd_img_request_get(struct rbd_img_request *img_request) -{ - dout("%s: img %p (was %d)\n", __func__, img_request, - kref_read(&img_request->kref)); - kref_get(&img_request->kref); -} - static void rbd_img_request_destroy(struct kref *kref); static void rbd_img_request_put(struct rbd_img_request *img_request) { @@ -1362,7 +1429,6 @@ static inline void rbd_img_obj_request_add(struct rbd_img_request *img_request, /* Image request now owns object's original reference */ obj_request->img_request = img_request; - img_request->pending_count++; dout("%s: img %p obj %p\n", __func__, img_request, obj_request); } @@ -1375,13 +1441,13 @@ static inline void rbd_img_obj_request_del(struct rbd_img_request *img_request, rbd_obj_request_put(obj_request); } -static void rbd_obj_request_submit(struct rbd_obj_request *obj_request) +static void rbd_osd_submit(struct ceph_osd_request *osd_req) { - struct ceph_osd_request *osd_req = obj_request->osd_req; + struct rbd_obj_request *obj_req = osd_req->r_priv; - dout("%s %p object_no %016llx %llu~%llu osd_req %p\n", __func__, - obj_request, obj_request->ex.oe_objno, obj_request->ex.oe_off, - obj_request->ex.oe_len, osd_req); + dout("%s osd_req %p for obj_req %p objno %llu %llu~%llu\n", + __func__, osd_req, obj_req, obj_req->ex.oe_objno, + obj_req->ex.oe_off, obj_req->ex.oe_len); ceph_osdc_start_request(osd_req->r_osdc, osd_req, false); } @@ -1457,41 +1523,38 @@ static bool rbd_img_is_write(struct rbd_img_request *img_req) } } -static void rbd_obj_handle_request(struct rbd_obj_request *obj_req); - static void rbd_osd_req_callback(struct ceph_osd_request *osd_req) { struct rbd_obj_request *obj_req = osd_req->r_priv; + int result; dout("%s osd_req %p result %d for obj_req %p\n", __func__, osd_req, osd_req->r_result, obj_req); - rbd_assert(osd_req == obj_req->osd_req); - obj_req->result = osd_req->r_result < 0 ? osd_req->r_result : 0; - if (!obj_req->result && !rbd_img_is_write(obj_req->img_request)) - obj_req->xferred = osd_req->r_result; + /* + * Writes aren't allowed to return a data payload. In some + * guarded write cases (e.g. stat + zero on an empty object) + * a stat response makes it through, but we don't care. + */ + if (osd_req->r_result > 0 && rbd_img_is_write(obj_req->img_request)) + result = 0; else - /* - * Writes aren't allowed to return a data payload. In some - * guarded write cases (e.g. stat + zero on an empty object) - * a stat response makes it through, but we don't care. - */ - obj_req->xferred = 0; + result = osd_req->r_result; - rbd_obj_handle_request(obj_req); + rbd_obj_handle_request(obj_req, result); } -static void rbd_osd_req_format_read(struct rbd_obj_request *obj_request) +static void rbd_osd_format_read(struct ceph_osd_request *osd_req) { - struct ceph_osd_request *osd_req = obj_request->osd_req; + struct rbd_obj_request *obj_request = osd_req->r_priv; osd_req->r_flags = CEPH_OSD_FLAG_READ; osd_req->r_snapid = obj_request->img_request->snap_id; } -static void rbd_osd_req_format_write(struct rbd_obj_request *obj_request) +static void rbd_osd_format_write(struct ceph_osd_request *osd_req) { - struct ceph_osd_request *osd_req = obj_request->osd_req; + struct rbd_obj_request *obj_request = osd_req->r_priv; osd_req->r_flags = CEPH_OSD_FLAG_WRITE; ktime_get_real_ts64(&osd_req->r_mtime); @@ -1499,19 +1562,21 @@ static void rbd_osd_req_format_write(struct rbd_obj_request *obj_request) } static struct ceph_osd_request * -__rbd_osd_req_create(struct rbd_obj_request *obj_req, - struct ceph_snap_context *snapc, unsigned int num_ops) +__rbd_obj_add_osd_request(struct rbd_obj_request *obj_req, + struct ceph_snap_context *snapc, int num_ops) { struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc; struct ceph_osd_request *req; const char *name_format = rbd_dev->image_format == 1 ? RBD_V1_DATA_FORMAT : RBD_V2_DATA_FORMAT; + int ret; req = ceph_osdc_alloc_request(osdc, snapc, num_ops, false, GFP_NOIO); if (!req) - return NULL; + return ERR_PTR(-ENOMEM); + list_add_tail(&req->r_private_item, &obj_req->osd_reqs); req->r_callback = rbd_osd_req_callback; req->r_priv = obj_req; @@ -1522,27 +1587,20 @@ __rbd_osd_req_create(struct rbd_obj_request *obj_req, ceph_oloc_copy(&req->r_base_oloc, &rbd_dev->header_oloc); req->r_base_oloc.pool = rbd_dev->layout.pool_id; - if (ceph_oid_aprintf(&req->r_base_oid, GFP_NOIO, name_format, - rbd_dev->header.object_prefix, obj_req->ex.oe_objno)) - goto err_req; + ret = ceph_oid_aprintf(&req->r_base_oid, GFP_NOIO, name_format, + rbd_dev->header.object_prefix, + obj_req->ex.oe_objno); + if (ret) + return ERR_PTR(ret); return req; - -err_req: - ceph_osdc_put_request(req); - return NULL; } static struct ceph_osd_request * -rbd_osd_req_create(struct rbd_obj_request *obj_req, unsigned int num_ops) +rbd_obj_add_osd_request(struct rbd_obj_request *obj_req, int num_ops) { - return __rbd_osd_req_create(obj_req, obj_req->img_request->snapc, - num_ops); -} - -static void rbd_osd_req_destroy(struct ceph_osd_request *osd_req) -{ - ceph_osdc_put_request(osd_req); + return __rbd_obj_add_osd_request(obj_req, obj_req->img_request->snapc, + num_ops); } static struct rbd_obj_request *rbd_obj_request_create(void) @@ -1554,6 +1612,8 @@ static struct rbd_obj_request *rbd_obj_request_create(void) return NULL; ceph_object_extent_init(&obj_request->ex); + INIT_LIST_HEAD(&obj_request->osd_reqs); + mutex_init(&obj_request->state_mutex); kref_init(&obj_request->kref); dout("%s %p\n", __func__, obj_request); @@ -1563,14 +1623,19 @@ static struct rbd_obj_request *rbd_obj_request_create(void) static void rbd_obj_request_destroy(struct kref *kref) { struct rbd_obj_request *obj_request; + struct ceph_osd_request *osd_req; u32 i; obj_request = container_of(kref, struct rbd_obj_request, kref); dout("%s: obj %p\n", __func__, obj_request); - if (obj_request->osd_req) - rbd_osd_req_destroy(obj_request->osd_req); + while (!list_empty(&obj_request->osd_reqs)) { + osd_req = list_first_entry(&obj_request->osd_reqs, + struct ceph_osd_request, r_private_item); + list_del_init(&osd_req->r_private_item); + ceph_osdc_put_request(osd_req); + } switch (obj_request->img_request->data_type) { case OBJ_REQUEST_NODATA: @@ -1684,8 +1749,9 @@ static struct rbd_img_request *rbd_img_request_create( if (rbd_dev_parent_get(rbd_dev)) img_request_layered_set(img_request); - spin_lock_init(&img_request->completion_lock); + INIT_LIST_HEAD(&img_request->lock_item); INIT_LIST_HEAD(&img_request->object_extents); + mutex_init(&img_request->state_mutex); kref_init(&img_request->kref); dout("%s: rbd_dev %p %s -> img %p\n", __func__, rbd_dev, @@ -1703,6 +1769,7 @@ static void rbd_img_request_destroy(struct kref *kref) dout("%s: img %p\n", __func__, img_request); + WARN_ON(!list_empty(&img_request->lock_item)); for_each_obj_request_safe(img_request, obj_request, next_obj_request) rbd_img_obj_request_del(img_request, obj_request); @@ -1717,6 +1784,466 @@ static void rbd_img_request_destroy(struct kref *kref) kmem_cache_free(rbd_img_request_cache, img_request); } +#define BITS_PER_OBJ 2 +#define OBJS_PER_BYTE (BITS_PER_BYTE / BITS_PER_OBJ) +#define OBJ_MASK ((1 << BITS_PER_OBJ) - 1) + +static void __rbd_object_map_index(struct rbd_device *rbd_dev, u64 objno, + u64 *index, u8 *shift) +{ + u32 off; + + rbd_assert(objno < rbd_dev->object_map_size); + *index = div_u64_rem(objno, OBJS_PER_BYTE, &off); + *shift = (OBJS_PER_BYTE - off - 1) * BITS_PER_OBJ; +} + +static u8 __rbd_object_map_get(struct rbd_device *rbd_dev, u64 objno) +{ + u64 index; + u8 shift; + + lockdep_assert_held(&rbd_dev->object_map_lock); + __rbd_object_map_index(rbd_dev, objno, &index, &shift); + return (rbd_dev->object_map[index] >> shift) & OBJ_MASK; +} + +static void __rbd_object_map_set(struct rbd_device *rbd_dev, u64 objno, u8 val) +{ + u64 index; + u8 shift; + u8 *p; + + lockdep_assert_held(&rbd_dev->object_map_lock); + rbd_assert(!(val & ~OBJ_MASK)); + + __rbd_object_map_index(rbd_dev, objno, &index, &shift); + p = &rbd_dev->object_map[index]; + *p = (*p & ~(OBJ_MASK << shift)) | (val << shift); +} + +static u8 rbd_object_map_get(struct rbd_device *rbd_dev, u64 objno) +{ + u8 state; + + spin_lock(&rbd_dev->object_map_lock); + state = __rbd_object_map_get(rbd_dev, objno); + spin_unlock(&rbd_dev->object_map_lock); + return state; +} + +static bool use_object_map(struct rbd_device *rbd_dev) +{ + return ((rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) && + !(rbd_dev->object_map_flags & RBD_FLAG_OBJECT_MAP_INVALID)); +} + +static bool rbd_object_map_may_exist(struct rbd_device *rbd_dev, u64 objno) +{ + u8 state; + + /* fall back to default logic if object map is disabled or invalid */ + if (!use_object_map(rbd_dev)) + return true; + + state = rbd_object_map_get(rbd_dev, objno); + return state != OBJECT_NONEXISTENT; +} + +static void rbd_object_map_name(struct rbd_device *rbd_dev, u64 snap_id, + struct ceph_object_id *oid) +{ + if (snap_id == CEPH_NOSNAP) + ceph_oid_printf(oid, "%s%s", RBD_OBJECT_MAP_PREFIX, + rbd_dev->spec->image_id); + else + ceph_oid_printf(oid, "%s%s.%016llx", RBD_OBJECT_MAP_PREFIX, + rbd_dev->spec->image_id, snap_id); +} + +static int rbd_object_map_lock(struct rbd_device *rbd_dev) +{ + struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc; + CEPH_DEFINE_OID_ONSTACK(oid); + u8 lock_type; + char *lock_tag; + struct ceph_locker *lockers; + u32 num_lockers; + bool broke_lock = false; + int ret; + + rbd_object_map_name(rbd_dev, CEPH_NOSNAP, &oid); + +again: + ret = ceph_cls_lock(osdc, &oid, &rbd_dev->header_oloc, RBD_LOCK_NAME, + CEPH_CLS_LOCK_EXCLUSIVE, "", "", "", 0); + if (ret != -EBUSY || broke_lock) { + if (ret == -EEXIST) + ret = 0; /* already locked by myself */ + if (ret) + rbd_warn(rbd_dev, "failed to lock object map: %d", ret); + return ret; + } + + ret = ceph_cls_lock_info(osdc, &oid, &rbd_dev->header_oloc, + RBD_LOCK_NAME, &lock_type, &lock_tag, + &lockers, &num_lockers); + if (ret) { + if (ret == -ENOENT) + goto again; + + rbd_warn(rbd_dev, "failed to get object map lockers: %d", ret); + return ret; + } + + kfree(lock_tag); + if (num_lockers == 0) + goto again; + + rbd_warn(rbd_dev, "breaking object map lock owned by %s%llu", + ENTITY_NAME(lockers[0].id.name)); + + ret = ceph_cls_break_lock(osdc, &oid, &rbd_dev->header_oloc, + RBD_LOCK_NAME, lockers[0].id.cookie, + &lockers[0].id.name); + ceph_free_lockers(lockers, num_lockers); + if (ret) { + if (ret == -ENOENT) + goto again; + + rbd_warn(rbd_dev, "failed to break object map lock: %d", ret); + return ret; + } + + broke_lock = true; + goto again; +} + +static void rbd_object_map_unlock(struct rbd_device *rbd_dev) +{ + struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc; + CEPH_DEFINE_OID_ONSTACK(oid); + int ret; + + rbd_object_map_name(rbd_dev, CEPH_NOSNAP, &oid); + + ret = ceph_cls_unlock(osdc, &oid, &rbd_dev->header_oloc, RBD_LOCK_NAME, + ""); + if (ret && ret != -ENOENT) + rbd_warn(rbd_dev, "failed to unlock object map: %d", ret); +} + +static int decode_object_map_header(void **p, void *end, u64 *object_map_size) +{ + u8 struct_v; + u32 struct_len; + u32 header_len; + void *header_end; + int ret; + + ceph_decode_32_safe(p, end, header_len, e_inval); + header_end = *p + header_len; + + ret = ceph_start_decoding(p, end, 1, "BitVector header", &struct_v, + &struct_len); + if (ret) + return ret; + + ceph_decode_64_safe(p, end, *object_map_size, e_inval); + + *p = header_end; + return 0; + +e_inval: + return -EINVAL; +} + +static int __rbd_object_map_load(struct rbd_device *rbd_dev) +{ + struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc; + CEPH_DEFINE_OID_ONSTACK(oid); + struct page **pages; + void *p, *end; + size_t reply_len; + u64 num_objects; + u64 object_map_bytes; + u64 object_map_size; + int num_pages; + int ret; + + rbd_assert(!rbd_dev->object_map && !rbd_dev->object_map_size); + + num_objects = ceph_get_num_objects(&rbd_dev->layout, + rbd_dev->mapping.size); + object_map_bytes = DIV_ROUND_UP_ULL(num_objects * BITS_PER_OBJ, + BITS_PER_BYTE); + num_pages = calc_pages_for(0, object_map_bytes) + 1; + pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL); + if (IS_ERR(pages)) + return PTR_ERR(pages); + + reply_len = num_pages * PAGE_SIZE; + rbd_object_map_name(rbd_dev, rbd_dev->spec->snap_id, &oid); + ret = ceph_osdc_call(osdc, &oid, &rbd_dev->header_oloc, + "rbd", "object_map_load", CEPH_OSD_FLAG_READ, + NULL, 0, pages, &reply_len); + if (ret) + goto out; + + p = page_address(pages[0]); + end = p + min(reply_len, (size_t)PAGE_SIZE); + ret = decode_object_map_header(&p, end, &object_map_size); + if (ret) + goto out; + + if (object_map_size != num_objects) { + rbd_warn(rbd_dev, "object map size mismatch: %llu vs %llu", + object_map_size, num_objects); + ret = -EINVAL; + goto out; + } + + if (offset_in_page(p) + object_map_bytes > reply_len) { + ret = -EINVAL; + goto out; + } + + rbd_dev->object_map = kvmalloc(object_map_bytes, GFP_KERNEL); + if (!rbd_dev->object_map) { + ret = -ENOMEM; + goto out; + } + + rbd_dev->object_map_size = object_map_size; + ceph_copy_from_page_vector(pages, rbd_dev->object_map, + offset_in_page(p), object_map_bytes); + +out: + ceph_release_page_vector(pages, num_pages); + return ret; +} + +static void rbd_object_map_free(struct rbd_device *rbd_dev) +{ + kvfree(rbd_dev->object_map); + rbd_dev->object_map = NULL; + rbd_dev->object_map_size = 0; +} + +static int rbd_object_map_load(struct rbd_device *rbd_dev) +{ + int ret; + + ret = __rbd_object_map_load(rbd_dev); + if (ret) + return ret; + + ret = rbd_dev_v2_get_flags(rbd_dev); + if (ret) { + rbd_object_map_free(rbd_dev); + return ret; + } + + if (rbd_dev->object_map_flags & RBD_FLAG_OBJECT_MAP_INVALID) + rbd_warn(rbd_dev, "object map is invalid"); + + return 0; +} + +static int rbd_object_map_open(struct rbd_device *rbd_dev) +{ + int ret; + + ret = rbd_object_map_lock(rbd_dev); + if (ret) + return ret; + + ret = rbd_object_map_load(rbd_dev); + if (ret) { + rbd_object_map_unlock(rbd_dev); + return ret; + } + + return 0; +} + +static void rbd_object_map_close(struct rbd_device *rbd_dev) +{ + rbd_object_map_free(rbd_dev); + rbd_object_map_unlock(rbd_dev); +} + +/* + * This function needs snap_id (or more precisely just something to + * distinguish between HEAD and snapshot object maps), new_state and + * current_state that were passed to rbd_object_map_update(). + * + * To avoid allocating and stashing a context we piggyback on the OSD + * request. A HEAD update has two ops (assert_locked). For new_state + * and current_state we decode our own object_map_update op, encoded in + * rbd_cls_object_map_update(). + */ +static int rbd_object_map_update_finish(struct rbd_obj_request *obj_req, + struct ceph_osd_request *osd_req) +{ + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + struct ceph_osd_data *osd_data; + u64 objno; + u8 state, new_state, current_state; + bool has_current_state; + void *p; + + if (osd_req->r_result) + return osd_req->r_result; + + /* + * Nothing to do for a snapshot object map. + */ + if (osd_req->r_num_ops == 1) + return 0; + + /* + * Update in-memory HEAD object map. + */ + rbd_assert(osd_req->r_num_ops == 2); + osd_data = osd_req_op_data(osd_req, 1, cls, request_data); + rbd_assert(osd_data->type == CEPH_OSD_DATA_TYPE_PAGES); + + p = page_address(osd_data->pages[0]); + objno = ceph_decode_64(&p); + rbd_assert(objno == obj_req->ex.oe_objno); + rbd_assert(ceph_decode_64(&p) == objno + 1); + new_state = ceph_decode_8(&p); + has_current_state = ceph_decode_8(&p); + if (has_current_state) + current_state = ceph_decode_8(&p); + + spin_lock(&rbd_dev->object_map_lock); + state = __rbd_object_map_get(rbd_dev, objno); + if (!has_current_state || current_state == state || + (current_state == OBJECT_EXISTS && state == OBJECT_EXISTS_CLEAN)) + __rbd_object_map_set(rbd_dev, objno, new_state); + spin_unlock(&rbd_dev->object_map_lock); + + return 0; +} + +static void rbd_object_map_callback(struct ceph_osd_request *osd_req) +{ + struct rbd_obj_request *obj_req = osd_req->r_priv; + int result; + + dout("%s osd_req %p result %d for obj_req %p\n", __func__, osd_req, + osd_req->r_result, obj_req); + + result = rbd_object_map_update_finish(obj_req, osd_req); + rbd_obj_handle_request(obj_req, result); +} + +static bool update_needed(struct rbd_device *rbd_dev, u64 objno, u8 new_state) +{ + u8 state = rbd_object_map_get(rbd_dev, objno); + + if (state == new_state || + (new_state == OBJECT_PENDING && state == OBJECT_NONEXISTENT) || + (new_state == OBJECT_NONEXISTENT && state != OBJECT_PENDING)) + return false; + + return true; +} + +static int rbd_cls_object_map_update(struct ceph_osd_request *req, + int which, u64 objno, u8 new_state, + const u8 *current_state) +{ + struct page **pages; + void *p, *start; + int ret; + + ret = osd_req_op_cls_init(req, which, "rbd", "object_map_update"); + if (ret) + return ret; + + pages = ceph_alloc_page_vector(1, GFP_NOIO); + if (IS_ERR(pages)) + return PTR_ERR(pages); + + p = start = page_address(pages[0]); + ceph_encode_64(&p, objno); + ceph_encode_64(&p, objno + 1); + ceph_encode_8(&p, new_state); + if (current_state) { + ceph_encode_8(&p, 1); + ceph_encode_8(&p, *current_state); + } else { + ceph_encode_8(&p, 0); + } + + osd_req_op_cls_request_data_pages(req, which, pages, p - start, 0, + false, true); + return 0; +} + +/* + * Return: + * 0 - object map update sent + * 1 - object map update isn't needed + * <0 - error + */ +static int rbd_object_map_update(struct rbd_obj_request *obj_req, u64 snap_id, + u8 new_state, const u8 *current_state) +{ + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc; + struct ceph_osd_request *req; + int num_ops = 1; + int which = 0; + int ret; + + if (snap_id == CEPH_NOSNAP) { + if (!update_needed(rbd_dev, obj_req->ex.oe_objno, new_state)) + return 1; + + num_ops++; /* assert_locked */ + } + + req = ceph_osdc_alloc_request(osdc, NULL, num_ops, false, GFP_NOIO); + if (!req) + return -ENOMEM; + + list_add_tail(&req->r_private_item, &obj_req->osd_reqs); + req->r_callback = rbd_object_map_callback; + req->r_priv = obj_req; + + rbd_object_map_name(rbd_dev, snap_id, &req->r_base_oid); + ceph_oloc_copy(&req->r_base_oloc, &rbd_dev->header_oloc); + req->r_flags = CEPH_OSD_FLAG_WRITE; + ktime_get_real_ts64(&req->r_mtime); + + if (snap_id == CEPH_NOSNAP) { + /* + * Protect against possible race conditions during lock + * ownership transitions. + */ + ret = ceph_cls_assert_locked(req, which++, RBD_LOCK_NAME, + CEPH_CLS_LOCK_EXCLUSIVE, "", ""); + if (ret) + return ret; + } + + ret = rbd_cls_object_map_update(req, which, obj_req->ex.oe_objno, + new_state, current_state); + if (ret) + return ret; + + ret = ceph_osdc_alloc_messages(req, GFP_NOIO); + if (ret) + return ret; + + ceph_osdc_start_request(osdc, req, false); + return 0; +} + static void prune_extents(struct ceph_file_extent *img_extents, u32 *num_img_extents, u64 overlap) { @@ -1764,11 +2291,13 @@ static int rbd_obj_calc_img_extents(struct rbd_obj_request *obj_req, return 0; } -static void rbd_osd_req_setup_data(struct rbd_obj_request *obj_req, u32 which) +static void rbd_osd_setup_data(struct ceph_osd_request *osd_req, int which) { + struct rbd_obj_request *obj_req = osd_req->r_priv; + switch (obj_req->img_request->data_type) { case OBJ_REQUEST_BIO: - osd_req_op_extent_osd_data_bio(obj_req->osd_req, which, + osd_req_op_extent_osd_data_bio(osd_req, which, &obj_req->bio_pos, obj_req->ex.oe_len); break; @@ -1777,7 +2306,7 @@ static void rbd_osd_req_setup_data(struct rbd_obj_request *obj_req, u32 which) rbd_assert(obj_req->bvec_pos.iter.bi_size == obj_req->ex.oe_len); rbd_assert(obj_req->bvec_idx == obj_req->bvec_count); - osd_req_op_extent_osd_data_bvec_pos(obj_req->osd_req, which, + osd_req_op_extent_osd_data_bvec_pos(osd_req, which, &obj_req->bvec_pos); break; default: @@ -1785,22 +2314,7 @@ static void rbd_osd_req_setup_data(struct rbd_obj_request *obj_req, u32 which) } } -static int rbd_obj_setup_read(struct rbd_obj_request *obj_req) -{ - obj_req->osd_req = __rbd_osd_req_create(obj_req, NULL, 1); - if (!obj_req->osd_req) - return -ENOMEM; - - osd_req_op_extent_init(obj_req->osd_req, 0, CEPH_OSD_OP_READ, - obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0); - rbd_osd_req_setup_data(obj_req, 0); - - rbd_osd_req_format_read(obj_req); - return 0; -} - -static int __rbd_obj_setup_stat(struct rbd_obj_request *obj_req, - unsigned int which) +static int rbd_osd_setup_stat(struct ceph_osd_request *osd_req, int which) { struct page **pages; @@ -1816,45 +2330,60 @@ static int __rbd_obj_setup_stat(struct rbd_obj_request *obj_req, if (IS_ERR(pages)) return PTR_ERR(pages); - osd_req_op_init(obj_req->osd_req, which, CEPH_OSD_OP_STAT, 0); - osd_req_op_raw_data_in_pages(obj_req->osd_req, which, pages, + osd_req_op_init(osd_req, which, CEPH_OSD_OP_STAT, 0); + osd_req_op_raw_data_in_pages(osd_req, which, pages, 8 + sizeof(struct ceph_timespec), 0, false, true); return 0; } -static int count_write_ops(struct rbd_obj_request *obj_req) +static int rbd_osd_setup_copyup(struct ceph_osd_request *osd_req, int which, + u32 bytes) +{ + struct rbd_obj_request *obj_req = osd_req->r_priv; + int ret; + + ret = osd_req_op_cls_init(osd_req, which, "rbd", "copyup"); + if (ret) + return ret; + + osd_req_op_cls_request_data_bvecs(osd_req, which, obj_req->copyup_bvecs, + obj_req->copyup_bvec_count, bytes); + return 0; +} + +static int rbd_obj_init_read(struct rbd_obj_request *obj_req) { - return 2; /* setallochint + write/writefull */ + obj_req->read_state = RBD_OBJ_READ_START; + return 0; } -static void __rbd_obj_setup_write(struct rbd_obj_request *obj_req, - unsigned int which) +static void __rbd_osd_setup_write_ops(struct ceph_osd_request *osd_req, + int which) { + struct rbd_obj_request *obj_req = osd_req->r_priv; struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; u16 opcode; - osd_req_op_alloc_hint_init(obj_req->osd_req, which++, - rbd_dev->layout.object_size, - rbd_dev->layout.object_size); + if (!use_object_map(rbd_dev) || + !(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST)) { + osd_req_op_alloc_hint_init(osd_req, which++, + rbd_dev->layout.object_size, + rbd_dev->layout.object_size); + } if (rbd_obj_is_entire(obj_req)) opcode = CEPH_OSD_OP_WRITEFULL; else opcode = CEPH_OSD_OP_WRITE; - osd_req_op_extent_init(obj_req->osd_req, which, opcode, + osd_req_op_extent_init(osd_req, which, opcode, obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0); - rbd_osd_req_setup_data(obj_req, which++); - - rbd_assert(which == obj_req->osd_req->r_num_ops); - rbd_osd_req_format_write(obj_req); + rbd_osd_setup_data(osd_req, which); } -static int rbd_obj_setup_write(struct rbd_obj_request *obj_req) +static int rbd_obj_init_write(struct rbd_obj_request *obj_req) { - unsigned int num_osd_ops, which = 0; - bool need_guard; int ret; /* reverse map the entire object onto the parent */ @@ -1862,24 +2391,10 @@ static int rbd_obj_setup_write(struct rbd_obj_request *obj_req) if (ret) return ret; - need_guard = rbd_obj_copyup_enabled(obj_req); - num_osd_ops = need_guard + count_write_ops(obj_req); - - obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops); - if (!obj_req->osd_req) - return -ENOMEM; - - if (need_guard) { - ret = __rbd_obj_setup_stat(obj_req, which++); - if (ret) - return ret; + if (rbd_obj_copyup_enabled(obj_req)) + obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ENABLED; - obj_req->write_state = RBD_OBJ_WRITE_GUARD; - } else { - obj_req->write_state = RBD_OBJ_WRITE_FLAT; - } - - __rbd_obj_setup_write(obj_req, which); + obj_req->write_state = RBD_OBJ_WRITE_START; return 0; } @@ -1889,11 +2404,26 @@ static u16 truncate_or_zero_opcode(struct rbd_obj_request *obj_req) CEPH_OSD_OP_ZERO; } -static int rbd_obj_setup_discard(struct rbd_obj_request *obj_req) +static void __rbd_osd_setup_discard_ops(struct ceph_osd_request *osd_req, + int which) +{ + struct rbd_obj_request *obj_req = osd_req->r_priv; + + if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents) { + rbd_assert(obj_req->flags & RBD_OBJ_FLAG_DELETION); + osd_req_op_init(osd_req, which, CEPH_OSD_OP_DELETE, 0); + } else { + osd_req_op_extent_init(osd_req, which, + truncate_or_zero_opcode(obj_req), + obj_req->ex.oe_off, obj_req->ex.oe_len, + 0, 0); + } +} + +static int rbd_obj_init_discard(struct rbd_obj_request *obj_req) { struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; - u64 off = obj_req->ex.oe_off; - u64 next_off = obj_req->ex.oe_off + obj_req->ex.oe_len; + u64 off, next_off; int ret; /* @@ -1906,10 +2436,17 @@ static int rbd_obj_setup_discard(struct rbd_obj_request *obj_req) */ if (rbd_dev->opts->alloc_size != rbd_dev->layout.object_size || !rbd_obj_is_tail(obj_req)) { - off = round_up(off, rbd_dev->opts->alloc_size); - next_off = round_down(next_off, rbd_dev->opts->alloc_size); + off = round_up(obj_req->ex.oe_off, rbd_dev->opts->alloc_size); + next_off = round_down(obj_req->ex.oe_off + obj_req->ex.oe_len, + rbd_dev->opts->alloc_size); if (off >= next_off) return 1; + + dout("%s %p %llu~%llu -> %llu~%llu\n", __func__, + obj_req, obj_req->ex.oe_off, obj_req->ex.oe_len, + off, next_off - off); + obj_req->ex.oe_off = off; + obj_req->ex.oe_len = next_off - off; } /* reverse map the entire object onto the parent */ @@ -1917,52 +2454,29 @@ static int rbd_obj_setup_discard(struct rbd_obj_request *obj_req) if (ret) return ret; - obj_req->osd_req = rbd_osd_req_create(obj_req, 1); - if (!obj_req->osd_req) - return -ENOMEM; - - if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents) { - osd_req_op_init(obj_req->osd_req, 0, CEPH_OSD_OP_DELETE, 0); - } else { - dout("%s %p %llu~%llu -> %llu~%llu\n", __func__, - obj_req, obj_req->ex.oe_off, obj_req->ex.oe_len, - off, next_off - off); - osd_req_op_extent_init(obj_req->osd_req, 0, - truncate_or_zero_opcode(obj_req), - off, next_off - off, 0, 0); - } + obj_req->flags |= RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT; + if (rbd_obj_is_entire(obj_req) && !obj_req->num_img_extents) + obj_req->flags |= RBD_OBJ_FLAG_DELETION; - obj_req->write_state = RBD_OBJ_WRITE_FLAT; - rbd_osd_req_format_write(obj_req); + obj_req->write_state = RBD_OBJ_WRITE_START; return 0; } -static int count_zeroout_ops(struct rbd_obj_request *obj_req) -{ - int num_osd_ops; - - if (rbd_obj_is_entire(obj_req) && obj_req->num_img_extents && - !rbd_obj_copyup_enabled(obj_req)) - num_osd_ops = 2; /* create + truncate */ - else - num_osd_ops = 1; /* delete/truncate/zero */ - - return num_osd_ops; -} - -static void __rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req, - unsigned int which) +static void __rbd_osd_setup_zeroout_ops(struct ceph_osd_request *osd_req, + int which) { + struct rbd_obj_request *obj_req = osd_req->r_priv; u16 opcode; if (rbd_obj_is_entire(obj_req)) { if (obj_req->num_img_extents) { - if (!rbd_obj_copyup_enabled(obj_req)) - osd_req_op_init(obj_req->osd_req, which++, + if (!(obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED)) + osd_req_op_init(osd_req, which++, CEPH_OSD_OP_CREATE, 0); opcode = CEPH_OSD_OP_TRUNCATE; } else { - osd_req_op_init(obj_req->osd_req, which++, + rbd_assert(obj_req->flags & RBD_OBJ_FLAG_DELETION); + osd_req_op_init(osd_req, which++, CEPH_OSD_OP_DELETE, 0); opcode = 0; } @@ -1971,18 +2485,13 @@ static void __rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req, } if (opcode) - osd_req_op_extent_init(obj_req->osd_req, which++, opcode, + osd_req_op_extent_init(osd_req, which, opcode, obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0); - - rbd_assert(which == obj_req->osd_req->r_num_ops); - rbd_osd_req_format_write(obj_req); } -static int rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req) +static int rbd_obj_init_zeroout(struct rbd_obj_request *obj_req) { - unsigned int num_osd_ops, which = 0; - bool need_guard; int ret; /* reverse map the entire object onto the parent */ @@ -1990,31 +2499,66 @@ static int rbd_obj_setup_zeroout(struct rbd_obj_request *obj_req) if (ret) return ret; - need_guard = rbd_obj_copyup_enabled(obj_req); - num_osd_ops = need_guard + count_zeroout_ops(obj_req); + if (rbd_obj_copyup_enabled(obj_req)) + obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ENABLED; + if (!obj_req->num_img_extents) { + obj_req->flags |= RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT; + if (rbd_obj_is_entire(obj_req)) + obj_req->flags |= RBD_OBJ_FLAG_DELETION; + } - obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops); - if (!obj_req->osd_req) - return -ENOMEM; + obj_req->write_state = RBD_OBJ_WRITE_START; + return 0; +} - if (need_guard) { - ret = __rbd_obj_setup_stat(obj_req, which++); - if (ret) - return ret; +static int count_write_ops(struct rbd_obj_request *obj_req) +{ + struct rbd_img_request *img_req = obj_req->img_request; - obj_req->write_state = RBD_OBJ_WRITE_GUARD; - } else { - obj_req->write_state = RBD_OBJ_WRITE_FLAT; + switch (img_req->op_type) { + case OBJ_OP_WRITE: + if (!use_object_map(img_req->rbd_dev) || + !(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST)) + return 2; /* setallochint + write/writefull */ + + return 1; /* write/writefull */ + case OBJ_OP_DISCARD: + return 1; /* delete/truncate/zero */ + case OBJ_OP_ZEROOUT: + if (rbd_obj_is_entire(obj_req) && obj_req->num_img_extents && + !(obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED)) + return 2; /* create + truncate */ + + return 1; /* delete/truncate/zero */ + default: + BUG(); } +} - __rbd_obj_setup_zeroout(obj_req, which); - return 0; +static void rbd_osd_setup_write_ops(struct ceph_osd_request *osd_req, + int which) +{ + struct rbd_obj_request *obj_req = osd_req->r_priv; + + switch (obj_req->img_request->op_type) { + case OBJ_OP_WRITE: + __rbd_osd_setup_write_ops(osd_req, which); + break; + case OBJ_OP_DISCARD: + __rbd_osd_setup_discard_ops(osd_req, which); + break; + case OBJ_OP_ZEROOUT: + __rbd_osd_setup_zeroout_ops(osd_req, which); + break; + default: + BUG(); + } } /* - * For each object request in @img_req, allocate an OSD request, add - * individual OSD ops and prepare them for submission. The number of - * OSD ops depends on op_type and the overlap point (if any). + * Prune the list of object requests (adjust offset and/or length, drop + * redundant requests). Prepare object request state machines and image + * request state machine for execution. */ static int __rbd_img_fill_request(struct rbd_img_request *img_req) { @@ -2024,16 +2568,16 @@ static int __rbd_img_fill_request(struct rbd_img_request *img_req) for_each_obj_request_safe(img_req, obj_req, next_obj_req) { switch (img_req->op_type) { case OBJ_OP_READ: - ret = rbd_obj_setup_read(obj_req); + ret = rbd_obj_init_read(obj_req); break; case OBJ_OP_WRITE: - ret = rbd_obj_setup_write(obj_req); + ret = rbd_obj_init_write(obj_req); break; case OBJ_OP_DISCARD: - ret = rbd_obj_setup_discard(obj_req); + ret = rbd_obj_init_discard(obj_req); break; case OBJ_OP_ZEROOUT: - ret = rbd_obj_setup_zeroout(obj_req); + ret = rbd_obj_init_zeroout(obj_req); break; default: BUG(); @@ -2041,17 +2585,12 @@ static int __rbd_img_fill_request(struct rbd_img_request *img_req) if (ret < 0) return ret; if (ret > 0) { - img_req->xferred += obj_req->ex.oe_len; - img_req->pending_count--; rbd_img_obj_request_del(img_req, obj_req); continue; } - - ret = ceph_osdc_alloc_messages(obj_req->osd_req, GFP_NOIO); - if (ret) - return ret; } + img_req->state = RBD_IMG_START; return 0; } @@ -2340,17 +2879,55 @@ static int rbd_img_fill_from_bvecs(struct rbd_img_request *img_req, &it); } -static void rbd_img_request_submit(struct rbd_img_request *img_request) +static void rbd_img_handle_request_work(struct work_struct *work) { - struct rbd_obj_request *obj_request; + struct rbd_img_request *img_req = + container_of(work, struct rbd_img_request, work); - dout("%s: img %p\n", __func__, img_request); + rbd_img_handle_request(img_req, img_req->work_result); +} - rbd_img_request_get(img_request); - for_each_obj_request(img_request, obj_request) - rbd_obj_request_submit(obj_request); +static void rbd_img_schedule(struct rbd_img_request *img_req, int result) +{ + INIT_WORK(&img_req->work, rbd_img_handle_request_work); + img_req->work_result = result; + queue_work(rbd_wq, &img_req->work); +} - rbd_img_request_put(img_request); +static bool rbd_obj_may_exist(struct rbd_obj_request *obj_req) +{ + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + + if (rbd_object_map_may_exist(rbd_dev, obj_req->ex.oe_objno)) { + obj_req->flags |= RBD_OBJ_FLAG_MAY_EXIST; + return true; + } + + dout("%s %p objno %llu assuming dne\n", __func__, obj_req, + obj_req->ex.oe_objno); + return false; +} + +static int rbd_obj_read_object(struct rbd_obj_request *obj_req) +{ + struct ceph_osd_request *osd_req; + int ret; + + osd_req = __rbd_obj_add_osd_request(obj_req, NULL, 1); + if (IS_ERR(osd_req)) + return PTR_ERR(osd_req); + + osd_req_op_extent_init(osd_req, 0, CEPH_OSD_OP_READ, + obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0); + rbd_osd_setup_data(osd_req, 0); + rbd_osd_format_read(osd_req); + + ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO); + if (ret) + return ret; + + rbd_osd_submit(osd_req); + return 0; } static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req) @@ -2396,51 +2973,144 @@ static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req) return ret; } - rbd_img_request_submit(child_img_req); + /* avoid parent chain recursion */ + rbd_img_schedule(child_img_req, 0); return 0; } -static bool rbd_obj_handle_read(struct rbd_obj_request *obj_req) +static bool rbd_obj_advance_read(struct rbd_obj_request *obj_req, int *result) { struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; int ret; - if (obj_req->result == -ENOENT && - rbd_dev->parent_overlap && !obj_req->tried_parent) { - /* reverse map this object extent onto the parent */ - ret = rbd_obj_calc_img_extents(obj_req, false); +again: + switch (obj_req->read_state) { + case RBD_OBJ_READ_START: + rbd_assert(!*result); + + if (!rbd_obj_may_exist(obj_req)) { + *result = -ENOENT; + obj_req->read_state = RBD_OBJ_READ_OBJECT; + goto again; + } + + ret = rbd_obj_read_object(obj_req); if (ret) { - obj_req->result = ret; + *result = ret; return true; } - - if (obj_req->num_img_extents) { - obj_req->tried_parent = true; - ret = rbd_obj_read_from_parent(obj_req); + obj_req->read_state = RBD_OBJ_READ_OBJECT; + return false; + case RBD_OBJ_READ_OBJECT: + if (*result == -ENOENT && rbd_dev->parent_overlap) { + /* reverse map this object extent onto the parent */ + ret = rbd_obj_calc_img_extents(obj_req, false); if (ret) { - obj_req->result = ret; + *result = ret; return true; } - return false; + if (obj_req->num_img_extents) { + ret = rbd_obj_read_from_parent(obj_req); + if (ret) { + *result = ret; + return true; + } + obj_req->read_state = RBD_OBJ_READ_PARENT; + return false; + } + } + + /* + * -ENOENT means a hole in the image -- zero-fill the entire + * length of the request. A short read also implies zero-fill + * to the end of the request. + */ + if (*result == -ENOENT) { + rbd_obj_zero_range(obj_req, 0, obj_req->ex.oe_len); + *result = 0; + } else if (*result >= 0) { + if (*result < obj_req->ex.oe_len) + rbd_obj_zero_range(obj_req, *result, + obj_req->ex.oe_len - *result); + else + rbd_assert(*result == obj_req->ex.oe_len); + *result = 0; } + return true; + case RBD_OBJ_READ_PARENT: + return true; + default: + BUG(); } +} - /* - * -ENOENT means a hole in the image -- zero-fill the entire - * length of the request. A short read also implies zero-fill - * to the end of the request. In both cases we update xferred - * count to indicate the whole request was satisfied. - */ - if (obj_req->result == -ENOENT || - (!obj_req->result && obj_req->xferred < obj_req->ex.oe_len)) { - rbd_assert(!obj_req->xferred || !obj_req->result); - rbd_obj_zero_range(obj_req, obj_req->xferred, - obj_req->ex.oe_len - obj_req->xferred); - obj_req->result = 0; - obj_req->xferred = obj_req->ex.oe_len; +static bool rbd_obj_write_is_noop(struct rbd_obj_request *obj_req) +{ + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + + if (rbd_object_map_may_exist(rbd_dev, obj_req->ex.oe_objno)) + obj_req->flags |= RBD_OBJ_FLAG_MAY_EXIST; + + if (!(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST) && + (obj_req->flags & RBD_OBJ_FLAG_NOOP_FOR_NONEXISTENT)) { + dout("%s %p noop for nonexistent\n", __func__, obj_req); + return true; } - return true; + return false; +} + +/* + * Return: + * 0 - object map update sent + * 1 - object map update isn't needed + * <0 - error + */ +static int rbd_obj_write_pre_object_map(struct rbd_obj_request *obj_req) +{ + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + u8 new_state; + + if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) + return 1; + + if (obj_req->flags & RBD_OBJ_FLAG_DELETION) + new_state = OBJECT_PENDING; + else + new_state = OBJECT_EXISTS; + + return rbd_object_map_update(obj_req, CEPH_NOSNAP, new_state, NULL); +} + +static int rbd_obj_write_object(struct rbd_obj_request *obj_req) +{ + struct ceph_osd_request *osd_req; + int num_ops = count_write_ops(obj_req); + int which = 0; + int ret; + + if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED) + num_ops++; /* stat */ + + osd_req = rbd_obj_add_osd_request(obj_req, num_ops); + if (IS_ERR(osd_req)) + return PTR_ERR(osd_req); + + if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED) { + ret = rbd_osd_setup_stat(osd_req, which++); + if (ret) + return ret; + } + + rbd_osd_setup_write_ops(osd_req, which); + rbd_osd_format_write(osd_req); + + ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO); + if (ret) + return ret; + + rbd_osd_submit(osd_req); + return 0; } /* @@ -2463,123 +3133,67 @@ static bool is_zero_bvecs(struct bio_vec *bvecs, u32 bytes) #define MODS_ONLY U32_MAX -static int rbd_obj_issue_copyup_empty_snapc(struct rbd_obj_request *obj_req, - u32 bytes) +static int rbd_obj_copyup_empty_snapc(struct rbd_obj_request *obj_req, + u32 bytes) { + struct ceph_osd_request *osd_req; int ret; dout("%s obj_req %p bytes %u\n", __func__, obj_req, bytes); - rbd_assert(obj_req->osd_req->r_ops[0].op == CEPH_OSD_OP_STAT); rbd_assert(bytes > 0 && bytes != MODS_ONLY); - rbd_osd_req_destroy(obj_req->osd_req); - obj_req->osd_req = __rbd_osd_req_create(obj_req, &rbd_empty_snapc, 1); - if (!obj_req->osd_req) - return -ENOMEM; + osd_req = __rbd_obj_add_osd_request(obj_req, &rbd_empty_snapc, 1); + if (IS_ERR(osd_req)) + return PTR_ERR(osd_req); - ret = osd_req_op_cls_init(obj_req->osd_req, 0, "rbd", "copyup"); + ret = rbd_osd_setup_copyup(osd_req, 0, bytes); if (ret) return ret; - osd_req_op_cls_request_data_bvecs(obj_req->osd_req, 0, - obj_req->copyup_bvecs, - obj_req->copyup_bvec_count, - bytes); - rbd_osd_req_format_write(obj_req); + rbd_osd_format_write(osd_req); - ret = ceph_osdc_alloc_messages(obj_req->osd_req, GFP_NOIO); + ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO); if (ret) return ret; - rbd_obj_request_submit(obj_req); + rbd_osd_submit(osd_req); return 0; } -static int rbd_obj_issue_copyup_ops(struct rbd_obj_request *obj_req, u32 bytes) +static int rbd_obj_copyup_current_snapc(struct rbd_obj_request *obj_req, + u32 bytes) { - struct rbd_img_request *img_req = obj_req->img_request; - unsigned int num_osd_ops = (bytes != MODS_ONLY); - unsigned int which = 0; + struct ceph_osd_request *osd_req; + int num_ops = count_write_ops(obj_req); + int which = 0; int ret; dout("%s obj_req %p bytes %u\n", __func__, obj_req, bytes); - rbd_assert(obj_req->osd_req->r_ops[0].op == CEPH_OSD_OP_STAT || - obj_req->osd_req->r_ops[0].op == CEPH_OSD_OP_CALL); - rbd_osd_req_destroy(obj_req->osd_req); - switch (img_req->op_type) { - case OBJ_OP_WRITE: - num_osd_ops += count_write_ops(obj_req); - break; - case OBJ_OP_ZEROOUT: - num_osd_ops += count_zeroout_ops(obj_req); - break; - default: - BUG(); - } + if (bytes != MODS_ONLY) + num_ops++; /* copyup */ - obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops); - if (!obj_req->osd_req) - return -ENOMEM; + osd_req = rbd_obj_add_osd_request(obj_req, num_ops); + if (IS_ERR(osd_req)) + return PTR_ERR(osd_req); if (bytes != MODS_ONLY) { - ret = osd_req_op_cls_init(obj_req->osd_req, which, "rbd", - "copyup"); + ret = rbd_osd_setup_copyup(osd_req, which++, bytes); if (ret) return ret; - - osd_req_op_cls_request_data_bvecs(obj_req->osd_req, which++, - obj_req->copyup_bvecs, - obj_req->copyup_bvec_count, - bytes); } - switch (img_req->op_type) { - case OBJ_OP_WRITE: - __rbd_obj_setup_write(obj_req, which); - break; - case OBJ_OP_ZEROOUT: - __rbd_obj_setup_zeroout(obj_req, which); - break; - default: - BUG(); - } + rbd_osd_setup_write_ops(osd_req, which); + rbd_osd_format_write(osd_req); - ret = ceph_osdc_alloc_messages(obj_req->osd_req, GFP_NOIO); + ret = ceph_osdc_alloc_messages(osd_req, GFP_NOIO); if (ret) return ret; - rbd_obj_request_submit(obj_req); + rbd_osd_submit(osd_req); return 0; } -static int rbd_obj_issue_copyup(struct rbd_obj_request *obj_req, u32 bytes) -{ - /* - * Only send non-zero copyup data to save some I/O and network - * bandwidth -- zero copyup data is equivalent to the object not - * existing. - */ - if (is_zero_bvecs(obj_req->copyup_bvecs, bytes)) { - dout("%s obj_req %p detected zeroes\n", __func__, obj_req); - bytes = 0; - } - - if (obj_req->img_request->snapc->num_snaps && bytes > 0) { - /* - * Send a copyup request with an empty snapshot context to - * deep-copyup the object through all existing snapshots. - * A second request with the current snapshot context will be - * sent for the actual modification. - */ - obj_req->write_state = RBD_OBJ_WRITE_COPYUP_EMPTY_SNAPC; - return rbd_obj_issue_copyup_empty_snapc(obj_req, bytes); - } - - obj_req->write_state = RBD_OBJ_WRITE_COPYUP_OPS; - return rbd_obj_issue_copyup_ops(obj_req, bytes); -} - static int setup_copyup_bvecs(struct rbd_obj_request *obj_req, u64 obj_overlap) { u32 i; @@ -2608,7 +3222,12 @@ static int setup_copyup_bvecs(struct rbd_obj_request *obj_req, u64 obj_overlap) return 0; } -static int rbd_obj_handle_write_guard(struct rbd_obj_request *obj_req) +/* + * The target object doesn't exist. Read the data for the entire + * target object up to the overlap point (if any) from the parent, + * so we can use it for a copyup. + */ +static int rbd_obj_copyup_read_parent(struct rbd_obj_request *obj_req) { struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; int ret; @@ -2623,178 +3242,492 @@ static int rbd_obj_handle_write_guard(struct rbd_obj_request *obj_req) * request -- pass MODS_ONLY since the copyup isn't needed * anymore. */ - obj_req->write_state = RBD_OBJ_WRITE_COPYUP_OPS; - return rbd_obj_issue_copyup_ops(obj_req, MODS_ONLY); + return rbd_obj_copyup_current_snapc(obj_req, MODS_ONLY); } ret = setup_copyup_bvecs(obj_req, rbd_obj_img_extents_bytes(obj_req)); if (ret) return ret; - obj_req->write_state = RBD_OBJ_WRITE_READ_FROM_PARENT; return rbd_obj_read_from_parent(obj_req); } -static bool rbd_obj_handle_write(struct rbd_obj_request *obj_req) +static void rbd_obj_copyup_object_maps(struct rbd_obj_request *obj_req) { + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + struct ceph_snap_context *snapc = obj_req->img_request->snapc; + u8 new_state; + u32 i; int ret; - switch (obj_req->write_state) { - case RBD_OBJ_WRITE_GUARD: - rbd_assert(!obj_req->xferred); - if (obj_req->result == -ENOENT) { - /* - * The target object doesn't exist. Read the data for - * the entire target object up to the overlap point (if - * any) from the parent, so we can use it for a copyup. - */ - ret = rbd_obj_handle_write_guard(obj_req); - if (ret) { - obj_req->result = ret; - return true; - } - return false; + rbd_assert(!obj_req->pending.result && !obj_req->pending.num_pending); + + if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) + return; + + if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ZEROS) + return; + + for (i = 0; i < snapc->num_snaps; i++) { + if ((rbd_dev->header.features & RBD_FEATURE_FAST_DIFF) && + i + 1 < snapc->num_snaps) + new_state = OBJECT_EXISTS_CLEAN; + else + new_state = OBJECT_EXISTS; + + ret = rbd_object_map_update(obj_req, snapc->snaps[i], + new_state, NULL); + if (ret < 0) { + obj_req->pending.result = ret; + return; } - /* fall through */ - case RBD_OBJ_WRITE_FLAT: - case RBD_OBJ_WRITE_COPYUP_OPS: - if (!obj_req->result) - /* - * There is no such thing as a successful short - * write -- indicate the whole request was satisfied. - */ - obj_req->xferred = obj_req->ex.oe_len; - return true; - case RBD_OBJ_WRITE_READ_FROM_PARENT: - if (obj_req->result) - return true; - rbd_assert(obj_req->xferred); - ret = rbd_obj_issue_copyup(obj_req, obj_req->xferred); + rbd_assert(!ret); + obj_req->pending.num_pending++; + } +} + +static void rbd_obj_copyup_write_object(struct rbd_obj_request *obj_req) +{ + u32 bytes = rbd_obj_img_extents_bytes(obj_req); + int ret; + + rbd_assert(!obj_req->pending.result && !obj_req->pending.num_pending); + + /* + * Only send non-zero copyup data to save some I/O and network + * bandwidth -- zero copyup data is equivalent to the object not + * existing. + */ + if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ZEROS) + bytes = 0; + + if (obj_req->img_request->snapc->num_snaps && bytes > 0) { + /* + * Send a copyup request with an empty snapshot context to + * deep-copyup the object through all existing snapshots. + * A second request with the current snapshot context will be + * sent for the actual modification. + */ + ret = rbd_obj_copyup_empty_snapc(obj_req, bytes); + if (ret) { + obj_req->pending.result = ret; + return; + } + + obj_req->pending.num_pending++; + bytes = MODS_ONLY; + } + + ret = rbd_obj_copyup_current_snapc(obj_req, bytes); + if (ret) { + obj_req->pending.result = ret; + return; + } + + obj_req->pending.num_pending++; +} + +static bool rbd_obj_advance_copyup(struct rbd_obj_request *obj_req, int *result) +{ + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + int ret; + +again: + switch (obj_req->copyup_state) { + case RBD_OBJ_COPYUP_START: + rbd_assert(!*result); + + ret = rbd_obj_copyup_read_parent(obj_req); if (ret) { - obj_req->result = ret; - obj_req->xferred = 0; + *result = ret; return true; } + if (obj_req->num_img_extents) + obj_req->copyup_state = RBD_OBJ_COPYUP_READ_PARENT; + else + obj_req->copyup_state = RBD_OBJ_COPYUP_WRITE_OBJECT; return false; - case RBD_OBJ_WRITE_COPYUP_EMPTY_SNAPC: - if (obj_req->result) + case RBD_OBJ_COPYUP_READ_PARENT: + if (*result) return true; - obj_req->write_state = RBD_OBJ_WRITE_COPYUP_OPS; - ret = rbd_obj_issue_copyup_ops(obj_req, MODS_ONLY); - if (ret) { - obj_req->result = ret; + if (is_zero_bvecs(obj_req->copyup_bvecs, + rbd_obj_img_extents_bytes(obj_req))) { + dout("%s %p detected zeros\n", __func__, obj_req); + obj_req->flags |= RBD_OBJ_FLAG_COPYUP_ZEROS; + } + + rbd_obj_copyup_object_maps(obj_req); + if (!obj_req->pending.num_pending) { + *result = obj_req->pending.result; + obj_req->copyup_state = RBD_OBJ_COPYUP_OBJECT_MAPS; + goto again; + } + obj_req->copyup_state = __RBD_OBJ_COPYUP_OBJECT_MAPS; + return false; + case __RBD_OBJ_COPYUP_OBJECT_MAPS: + if (!pending_result_dec(&obj_req->pending, result)) + return false; + /* fall through */ + case RBD_OBJ_COPYUP_OBJECT_MAPS: + if (*result) { + rbd_warn(rbd_dev, "snap object map update failed: %d", + *result); return true; } + + rbd_obj_copyup_write_object(obj_req); + if (!obj_req->pending.num_pending) { + *result = obj_req->pending.result; + obj_req->copyup_state = RBD_OBJ_COPYUP_WRITE_OBJECT; + goto again; + } + obj_req->copyup_state = __RBD_OBJ_COPYUP_WRITE_OBJECT; return false; + case __RBD_OBJ_COPYUP_WRITE_OBJECT: + if (!pending_result_dec(&obj_req->pending, result)) + return false; + /* fall through */ + case RBD_OBJ_COPYUP_WRITE_OBJECT: + return true; default: BUG(); } } /* - * Returns true if @obj_req is completed, or false otherwise. + * Return: + * 0 - object map update sent + * 1 - object map update isn't needed + * <0 - error */ -static bool __rbd_obj_handle_request(struct rbd_obj_request *obj_req) +static int rbd_obj_write_post_object_map(struct rbd_obj_request *obj_req) { - switch (obj_req->img_request->op_type) { - case OBJ_OP_READ: - return rbd_obj_handle_read(obj_req); - case OBJ_OP_WRITE: - return rbd_obj_handle_write(obj_req); - case OBJ_OP_DISCARD: - case OBJ_OP_ZEROOUT: - if (rbd_obj_handle_write(obj_req)) { + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + u8 current_state = OBJECT_PENDING; + + if (!(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) + return 1; + + if (!(obj_req->flags & RBD_OBJ_FLAG_DELETION)) + return 1; + + return rbd_object_map_update(obj_req, CEPH_NOSNAP, OBJECT_NONEXISTENT, + ¤t_state); +} + +static bool rbd_obj_advance_write(struct rbd_obj_request *obj_req, int *result) +{ + struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev; + int ret; + +again: + switch (obj_req->write_state) { + case RBD_OBJ_WRITE_START: + rbd_assert(!*result); + + if (rbd_obj_write_is_noop(obj_req)) + return true; + + ret = rbd_obj_write_pre_object_map(obj_req); + if (ret < 0) { + *result = ret; + return true; + } + obj_req->write_state = RBD_OBJ_WRITE_PRE_OBJECT_MAP; + if (ret > 0) + goto again; + return false; + case RBD_OBJ_WRITE_PRE_OBJECT_MAP: + if (*result) { + rbd_warn(rbd_dev, "pre object map update failed: %d", + *result); + return true; + } + ret = rbd_obj_write_object(obj_req); + if (ret) { + *result = ret; + return true; + } + obj_req->write_state = RBD_OBJ_WRITE_OBJECT; + return false; + case RBD_OBJ_WRITE_OBJECT: + if (*result == -ENOENT) { + if (obj_req->flags & RBD_OBJ_FLAG_COPYUP_ENABLED) { + *result = 0; + obj_req->copyup_state = RBD_OBJ_COPYUP_START; + obj_req->write_state = __RBD_OBJ_WRITE_COPYUP; + goto again; + } /* - * Hide -ENOENT from delete/truncate/zero -- discarding - * a non-existent object is not a problem. + * On a non-existent object: + * delete - -ENOENT, truncate/zero - 0 */ - if (obj_req->result == -ENOENT) { - obj_req->result = 0; - obj_req->xferred = obj_req->ex.oe_len; - } + if (obj_req->flags & RBD_OBJ_FLAG_DELETION) + *result = 0; + } + if (*result) + return true; + + obj_req->write_state = RBD_OBJ_WRITE_COPYUP; + goto again; + case __RBD_OBJ_WRITE_COPYUP: + if (!rbd_obj_advance_copyup(obj_req, result)) + return false; + /* fall through */ + case RBD_OBJ_WRITE_COPYUP: + if (*result) { + rbd_warn(rbd_dev, "copyup failed: %d", *result); + return true; + } + ret = rbd_obj_write_post_object_map(obj_req); + if (ret < 0) { + *result = ret; return true; } + obj_req->write_state = RBD_OBJ_WRITE_POST_OBJECT_MAP; + if (ret > 0) + goto again; return false; + case RBD_OBJ_WRITE_POST_OBJECT_MAP: + if (*result) + rbd_warn(rbd_dev, "post object map update failed: %d", + *result); + return true; default: BUG(); } } -static void rbd_obj_end_request(struct rbd_obj_request *obj_req) +/* + * Return true if @obj_req is completed. + */ +static bool __rbd_obj_handle_request(struct rbd_obj_request *obj_req, + int *result) { struct rbd_img_request *img_req = obj_req->img_request; + struct rbd_device *rbd_dev = img_req->rbd_dev; + bool done; - rbd_assert((!obj_req->result && - obj_req->xferred == obj_req->ex.oe_len) || - (obj_req->result < 0 && !obj_req->xferred)); - if (!obj_req->result) { - img_req->xferred += obj_req->xferred; - return; - } + mutex_lock(&obj_req->state_mutex); + if (!rbd_img_is_write(img_req)) + done = rbd_obj_advance_read(obj_req, result); + else + done = rbd_obj_advance_write(obj_req, result); + mutex_unlock(&obj_req->state_mutex); - rbd_warn(img_req->rbd_dev, - "%s at objno %llu %llu~%llu result %d xferred %llu", - obj_op_name(img_req->op_type), obj_req->ex.oe_objno, - obj_req->ex.oe_off, obj_req->ex.oe_len, obj_req->result, - obj_req->xferred); - if (!img_req->result) { - img_req->result = obj_req->result; - img_req->xferred = 0; + if (done && *result) { + rbd_assert(*result < 0); + rbd_warn(rbd_dev, "%s at objno %llu %llu~%llu result %d", + obj_op_name(img_req->op_type), obj_req->ex.oe_objno, + obj_req->ex.oe_off, obj_req->ex.oe_len, *result); } + return done; } -static void rbd_img_end_child_request(struct rbd_img_request *img_req) +/* + * This is open-coded in rbd_img_handle_request() to avoid parent chain + * recursion. + */ +static void rbd_obj_handle_request(struct rbd_obj_request *obj_req, int result) { - struct rbd_obj_request *obj_req = img_req->obj_request; + if (__rbd_obj_handle_request(obj_req, &result)) + rbd_img_handle_request(obj_req->img_request, result); +} - rbd_assert(test_bit(IMG_REQ_CHILD, &img_req->flags)); - rbd_assert((!img_req->result && - img_req->xferred == rbd_obj_img_extents_bytes(obj_req)) || - (img_req->result < 0 && !img_req->xferred)); +static bool need_exclusive_lock(struct rbd_img_request *img_req) +{ + struct rbd_device *rbd_dev = img_req->rbd_dev; - obj_req->result = img_req->result; - obj_req->xferred = img_req->xferred; - rbd_img_request_put(img_req); + if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK)) + return false; + + if (rbd_dev->spec->snap_id != CEPH_NOSNAP) + return false; + + rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags)); + if (rbd_dev->opts->lock_on_read || + (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) + return true; + + return rbd_img_is_write(img_req); } -static void rbd_img_end_request(struct rbd_img_request *img_req) +static bool rbd_lock_add_request(struct rbd_img_request *img_req) { - rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags)); - rbd_assert((!img_req->result && - img_req->xferred == blk_rq_bytes(img_req->rq)) || - (img_req->result < 0 && !img_req->xferred)); + struct rbd_device *rbd_dev = img_req->rbd_dev; + bool locked; + + lockdep_assert_held(&rbd_dev->lock_rwsem); + locked = rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED; + spin_lock(&rbd_dev->lock_lists_lock); + rbd_assert(list_empty(&img_req->lock_item)); + if (!locked) + list_add_tail(&img_req->lock_item, &rbd_dev->acquiring_list); + else + list_add_tail(&img_req->lock_item, &rbd_dev->running_list); + spin_unlock(&rbd_dev->lock_lists_lock); + return locked; +} + +static void rbd_lock_del_request(struct rbd_img_request *img_req) +{ + struct rbd_device *rbd_dev = img_req->rbd_dev; + bool need_wakeup; - blk_mq_end_request(img_req->rq, - errno_to_blk_status(img_req->result)); - rbd_img_request_put(img_req); + lockdep_assert_held(&rbd_dev->lock_rwsem); + spin_lock(&rbd_dev->lock_lists_lock); + rbd_assert(!list_empty(&img_req->lock_item)); + list_del_init(&img_req->lock_item); + need_wakeup = (rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING && + list_empty(&rbd_dev->running_list)); + spin_unlock(&rbd_dev->lock_lists_lock); + if (need_wakeup) + complete(&rbd_dev->releasing_wait); } -static void rbd_obj_handle_request(struct rbd_obj_request *obj_req) +static int rbd_img_exclusive_lock(struct rbd_img_request *img_req) { - struct rbd_img_request *img_req; + struct rbd_device *rbd_dev = img_req->rbd_dev; + + if (!need_exclusive_lock(img_req)) + return 1; + + if (rbd_lock_add_request(img_req)) + return 1; + + if (rbd_dev->opts->exclusive) { + WARN_ON(1); /* lock got released? */ + return -EROFS; + } + + /* + * Note the use of mod_delayed_work() in rbd_acquire_lock() + * and cancel_delayed_work() in wake_lock_waiters(). + */ + dout("%s rbd_dev %p queueing lock_dwork\n", __func__, rbd_dev); + queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0); + return 0; +} + +static void rbd_img_object_requests(struct rbd_img_request *img_req) +{ + struct rbd_obj_request *obj_req; + + rbd_assert(!img_req->pending.result && !img_req->pending.num_pending); + + for_each_obj_request(img_req, obj_req) { + int result = 0; + + if (__rbd_obj_handle_request(obj_req, &result)) { + if (result) { + img_req->pending.result = result; + return; + } + } else { + img_req->pending.num_pending++; + } + } +} + +static bool rbd_img_advance(struct rbd_img_request *img_req, int *result) +{ + struct rbd_device *rbd_dev = img_req->rbd_dev; + int ret; again: - if (!__rbd_obj_handle_request(obj_req)) - return; + switch (img_req->state) { + case RBD_IMG_START: + rbd_assert(!*result); - img_req = obj_req->img_request; - spin_lock(&img_req->completion_lock); - rbd_obj_end_request(obj_req); - rbd_assert(img_req->pending_count); - if (--img_req->pending_count) { - spin_unlock(&img_req->completion_lock); - return; + ret = rbd_img_exclusive_lock(img_req); + if (ret < 0) { + *result = ret; + return true; + } + img_req->state = RBD_IMG_EXCLUSIVE_LOCK; + if (ret > 0) + goto again; + return false; + case RBD_IMG_EXCLUSIVE_LOCK: + if (*result) + return true; + + rbd_assert(!need_exclusive_lock(img_req) || + __rbd_is_lock_owner(rbd_dev)); + + rbd_img_object_requests(img_req); + if (!img_req->pending.num_pending) { + *result = img_req->pending.result; + img_req->state = RBD_IMG_OBJECT_REQUESTS; + goto again; + } + img_req->state = __RBD_IMG_OBJECT_REQUESTS; + return false; + case __RBD_IMG_OBJECT_REQUESTS: + if (!pending_result_dec(&img_req->pending, result)) + return false; + /* fall through */ + case RBD_IMG_OBJECT_REQUESTS: + return true; + default: + BUG(); + } +} + +/* + * Return true if @img_req is completed. + */ +static bool __rbd_img_handle_request(struct rbd_img_request *img_req, + int *result) +{ + struct rbd_device *rbd_dev = img_req->rbd_dev; + bool done; + + if (need_exclusive_lock(img_req)) { + down_read(&rbd_dev->lock_rwsem); + mutex_lock(&img_req->state_mutex); + done = rbd_img_advance(img_req, result); + if (done) + rbd_lock_del_request(img_req); + mutex_unlock(&img_req->state_mutex); + up_read(&rbd_dev->lock_rwsem); + } else { + mutex_lock(&img_req->state_mutex); + done = rbd_img_advance(img_req, result); + mutex_unlock(&img_req->state_mutex); + } + + if (done && *result) { + rbd_assert(*result < 0); + rbd_warn(rbd_dev, "%s%s result %d", + test_bit(IMG_REQ_CHILD, &img_req->flags) ? "child " : "", + obj_op_name(img_req->op_type), *result); } + return done; +} + +static void rbd_img_handle_request(struct rbd_img_request *img_req, int result) +{ +again: + if (!__rbd_img_handle_request(img_req, &result)) + return; - spin_unlock(&img_req->completion_lock); if (test_bit(IMG_REQ_CHILD, &img_req->flags)) { - obj_req = img_req->obj_request; - rbd_img_end_child_request(img_req); - goto again; + struct rbd_obj_request *obj_req = img_req->obj_request; + + rbd_img_request_put(img_req); + if (__rbd_obj_handle_request(obj_req, &result)) { + img_req = obj_req->img_request; + goto again; + } + } else { + struct request *rq = img_req->rq; + + rbd_img_request_put(img_req); + blk_mq_end_request(rq, errno_to_blk_status(result)); } - rbd_img_end_request(img_req); } static const struct rbd_client_id rbd_empty_cid; @@ -2839,6 +3772,7 @@ static void __rbd_lock(struct rbd_device *rbd_dev, const char *cookie) { struct rbd_client_id cid = rbd_get_cid(rbd_dev); + rbd_dev->lock_state = RBD_LOCK_STATE_LOCKED; strcpy(rbd_dev->lock_cookie, cookie); rbd_set_owner_cid(rbd_dev, &cid); queue_work(rbd_dev->task_wq, &rbd_dev->acquired_lock_work); @@ -2863,7 +3797,6 @@ static int rbd_lock(struct rbd_device *rbd_dev) if (ret) return ret; - rbd_dev->lock_state = RBD_LOCK_STATE_LOCKED; __rbd_lock(rbd_dev, cookie); return 0; } @@ -2882,7 +3815,7 @@ static void rbd_unlock(struct rbd_device *rbd_dev) ret = ceph_cls_unlock(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc, RBD_LOCK_NAME, rbd_dev->lock_cookie); if (ret && ret != -ENOENT) - rbd_warn(rbd_dev, "failed to unlock: %d", ret); + rbd_warn(rbd_dev, "failed to unlock header: %d", ret); /* treat errors as the image is unlocked */ rbd_dev->lock_state = RBD_LOCK_STATE_UNLOCKED; @@ -3009,15 +3942,34 @@ e_inval: goto out; } -static void wake_requests(struct rbd_device *rbd_dev, bool wake_all) +/* + * Either image request state machine(s) or rbd_add_acquire_lock() + * (i.e. "rbd map"). + */ +static void wake_lock_waiters(struct rbd_device *rbd_dev, int result) { - dout("%s rbd_dev %p wake_all %d\n", __func__, rbd_dev, wake_all); + struct rbd_img_request *img_req; + + dout("%s rbd_dev %p result %d\n", __func__, rbd_dev, result); + lockdep_assert_held_write(&rbd_dev->lock_rwsem); cancel_delayed_work(&rbd_dev->lock_dwork); - if (wake_all) - wake_up_all(&rbd_dev->lock_waitq); - else - wake_up(&rbd_dev->lock_waitq); + if (!completion_done(&rbd_dev->acquire_wait)) { + rbd_assert(list_empty(&rbd_dev->acquiring_list) && + list_empty(&rbd_dev->running_list)); + rbd_dev->acquire_err = result; + complete_all(&rbd_dev->acquire_wait); + return; + } + + list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) { + mutex_lock(&img_req->state_mutex); + rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK); + rbd_img_schedule(img_req, result); + mutex_unlock(&img_req->state_mutex); + } + + list_splice_tail_init(&rbd_dev->acquiring_list, &rbd_dev->running_list); } static int get_lock_owner_info(struct rbd_device *rbd_dev, @@ -3132,13 +4084,10 @@ static int rbd_try_lock(struct rbd_device *rbd_dev) goto again; ret = find_watcher(rbd_dev, lockers); - if (ret) { - if (ret > 0) - ret = 0; /* have to request lock */ - goto out; - } + if (ret) + goto out; /* request lock or error */ - rbd_warn(rbd_dev, "%s%llu seems dead, breaking lock", + rbd_warn(rbd_dev, "breaking header lock owned by %s%llu", ENTITY_NAME(lockers[0].id.name)); ret = ceph_monc_blacklist_add(&client->monc, @@ -3165,53 +4114,90 @@ out: return ret; } +static int rbd_post_acquire_action(struct rbd_device *rbd_dev) +{ + int ret; + + if (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) { + ret = rbd_object_map_open(rbd_dev); + if (ret) + return ret; + } + + return 0; +} + /* - * ret is set only if lock_state is RBD_LOCK_STATE_UNLOCKED + * Return: + * 0 - lock acquired + * 1 - caller should call rbd_request_lock() + * <0 - error */ -static enum rbd_lock_state rbd_try_acquire_lock(struct rbd_device *rbd_dev, - int *pret) +static int rbd_try_acquire_lock(struct rbd_device *rbd_dev) { - enum rbd_lock_state lock_state; + int ret; down_read(&rbd_dev->lock_rwsem); dout("%s rbd_dev %p read lock_state %d\n", __func__, rbd_dev, rbd_dev->lock_state); if (__rbd_is_lock_owner(rbd_dev)) { - lock_state = rbd_dev->lock_state; up_read(&rbd_dev->lock_rwsem); - return lock_state; + return 0; } up_read(&rbd_dev->lock_rwsem); down_write(&rbd_dev->lock_rwsem); dout("%s rbd_dev %p write lock_state %d\n", __func__, rbd_dev, rbd_dev->lock_state); - if (!__rbd_is_lock_owner(rbd_dev)) { - *pret = rbd_try_lock(rbd_dev); - if (*pret) - rbd_warn(rbd_dev, "failed to acquire lock: %d", *pret); + if (__rbd_is_lock_owner(rbd_dev)) { + up_write(&rbd_dev->lock_rwsem); + return 0; + } + + ret = rbd_try_lock(rbd_dev); + if (ret < 0) { + rbd_warn(rbd_dev, "failed to lock header: %d", ret); + if (ret == -EBLACKLISTED) + goto out; + + ret = 1; /* request lock anyway */ + } + if (ret > 0) { + up_write(&rbd_dev->lock_rwsem); + return ret; + } + + rbd_assert(rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED); + rbd_assert(list_empty(&rbd_dev->running_list)); + + ret = rbd_post_acquire_action(rbd_dev); + if (ret) { + rbd_warn(rbd_dev, "post-acquire action failed: %d", ret); + /* + * Can't stay in RBD_LOCK_STATE_LOCKED because + * rbd_lock_add_request() would let the request through, + * assuming that e.g. object map is locked and loaded. + */ + rbd_unlock(rbd_dev); } - lock_state = rbd_dev->lock_state; +out: + wake_lock_waiters(rbd_dev, ret); up_write(&rbd_dev->lock_rwsem); - return lock_state; + return ret; } static void rbd_acquire_lock(struct work_struct *work) { struct rbd_device *rbd_dev = container_of(to_delayed_work(work), struct rbd_device, lock_dwork); - enum rbd_lock_state lock_state; - int ret = 0; + int ret; dout("%s rbd_dev %p\n", __func__, rbd_dev); again: - lock_state = rbd_try_acquire_lock(rbd_dev, &ret); - if (lock_state != RBD_LOCK_STATE_UNLOCKED || ret == -EBLACKLISTED) { - if (lock_state == RBD_LOCK_STATE_LOCKED) - wake_requests(rbd_dev, true); - dout("%s rbd_dev %p lock_state %d ret %d - done\n", __func__, - rbd_dev, lock_state, ret); + ret = rbd_try_acquire_lock(rbd_dev); + if (ret <= 0) { + dout("%s rbd_dev %p ret %d - done\n", __func__, rbd_dev, ret); return; } @@ -3220,16 +4206,9 @@ again: goto again; /* treat this as a dead client */ } else if (ret == -EROFS) { rbd_warn(rbd_dev, "peer will not release lock"); - /* - * If this is rbd_add_acquire_lock(), we want to fail - * immediately -- reuse BLACKLISTED flag. Otherwise we - * want to block. - */ - if (!(rbd_dev->disk->flags & GENHD_FL_UP)) { - set_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags); - /* wake "rbd map --exclusive" process */ - wake_requests(rbd_dev, false); - } + down_write(&rbd_dev->lock_rwsem); + wake_lock_waiters(rbd_dev, ret); + up_write(&rbd_dev->lock_rwsem); } else if (ret < 0) { rbd_warn(rbd_dev, "error requesting lock: %d", ret); mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, @@ -3246,43 +4225,67 @@ again: } } -/* - * lock_rwsem must be held for write - */ -static bool rbd_release_lock(struct rbd_device *rbd_dev) +static bool rbd_quiesce_lock(struct rbd_device *rbd_dev) { - dout("%s rbd_dev %p read lock_state %d\n", __func__, rbd_dev, - rbd_dev->lock_state); + bool need_wait; + + dout("%s rbd_dev %p\n", __func__, rbd_dev); + lockdep_assert_held_write(&rbd_dev->lock_rwsem); + if (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED) return false; - rbd_dev->lock_state = RBD_LOCK_STATE_RELEASING; - downgrade_write(&rbd_dev->lock_rwsem); /* * Ensure that all in-flight IO is flushed. - * - * FIXME: ceph_osdc_sync() flushes the entire OSD client, which - * may be shared with other devices. */ - ceph_osdc_sync(&rbd_dev->rbd_client->client->osdc); + rbd_dev->lock_state = RBD_LOCK_STATE_RELEASING; + rbd_assert(!completion_done(&rbd_dev->releasing_wait)); + need_wait = !list_empty(&rbd_dev->running_list); + downgrade_write(&rbd_dev->lock_rwsem); + if (need_wait) + wait_for_completion(&rbd_dev->releasing_wait); up_read(&rbd_dev->lock_rwsem); down_write(&rbd_dev->lock_rwsem); - dout("%s rbd_dev %p write lock_state %d\n", __func__, rbd_dev, - rbd_dev->lock_state); if (rbd_dev->lock_state != RBD_LOCK_STATE_RELEASING) return false; + rbd_assert(list_empty(&rbd_dev->running_list)); + return true; +} + +static void rbd_pre_release_action(struct rbd_device *rbd_dev) +{ + if (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) + rbd_object_map_close(rbd_dev); +} + +static void __rbd_release_lock(struct rbd_device *rbd_dev) +{ + rbd_assert(list_empty(&rbd_dev->running_list)); + + rbd_pre_release_action(rbd_dev); rbd_unlock(rbd_dev); +} + +/* + * lock_rwsem must be held for write + */ +static void rbd_release_lock(struct rbd_device *rbd_dev) +{ + if (!rbd_quiesce_lock(rbd_dev)) + return; + + __rbd_release_lock(rbd_dev); + /* * Give others a chance to grab the lock - we would re-acquire - * almost immediately if we got new IO during ceph_osdc_sync() - * otherwise. We need to ack our own notifications, so this - * lock_dwork will be requeued from rbd_wait_state_locked() - * after wake_requests() in rbd_handle_released_lock(). + * almost immediately if we got new IO while draining the running + * list otherwise. We need to ack our own notifications, so this + * lock_dwork will be requeued from rbd_handle_released_lock() by + * way of maybe_kick_acquire(). */ cancel_delayed_work(&rbd_dev->lock_dwork); - return true; } static void rbd_release_lock_work(struct work_struct *work) @@ -3295,6 +4298,23 @@ static void rbd_release_lock_work(struct work_struct *work) up_write(&rbd_dev->lock_rwsem); } +static void maybe_kick_acquire(struct rbd_device *rbd_dev) +{ + bool have_requests; + + dout("%s rbd_dev %p\n", __func__, rbd_dev); + if (__rbd_is_lock_owner(rbd_dev)) + return; + + spin_lock(&rbd_dev->lock_lists_lock); + have_requests = !list_empty(&rbd_dev->acquiring_list); + spin_unlock(&rbd_dev->lock_lists_lock); + if (have_requests || delayed_work_pending(&rbd_dev->lock_dwork)) { + dout("%s rbd_dev %p kicking lock_dwork\n", __func__, rbd_dev); + mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0); + } +} + static void rbd_handle_acquired_lock(struct rbd_device *rbd_dev, u8 struct_v, void **p) { @@ -3324,8 +4344,7 @@ static void rbd_handle_acquired_lock(struct rbd_device *rbd_dev, u8 struct_v, down_read(&rbd_dev->lock_rwsem); } - if (!__rbd_is_lock_owner(rbd_dev)) - wake_requests(rbd_dev, false); + maybe_kick_acquire(rbd_dev); up_read(&rbd_dev->lock_rwsem); } @@ -3357,8 +4376,7 @@ static void rbd_handle_released_lock(struct rbd_device *rbd_dev, u8 struct_v, down_read(&rbd_dev->lock_rwsem); } - if (!__rbd_is_lock_owner(rbd_dev)) - wake_requests(rbd_dev, false); + maybe_kick_acquire(rbd_dev); up_read(&rbd_dev->lock_rwsem); } @@ -3608,7 +4626,6 @@ static void cancel_tasks_sync(struct rbd_device *rbd_dev) static void rbd_unregister_watch(struct rbd_device *rbd_dev) { - WARN_ON(waitqueue_active(&rbd_dev->lock_waitq)); cancel_tasks_sync(rbd_dev); mutex_lock(&rbd_dev->watch_mutex); @@ -3630,7 +4647,8 @@ static void rbd_reacquire_lock(struct rbd_device *rbd_dev) char cookie[32]; int ret; - WARN_ON(rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED); + if (!rbd_quiesce_lock(rbd_dev)) + return; format_lock_cookie(rbd_dev, cookie); ret = ceph_cls_set_cookie(osdc, &rbd_dev->header_oid, @@ -3646,11 +4664,11 @@ static void rbd_reacquire_lock(struct rbd_device *rbd_dev) * Lock cookie cannot be updated on older OSDs, so do * a manual release and queue an acquire. */ - if (rbd_release_lock(rbd_dev)) - queue_delayed_work(rbd_dev->task_wq, - &rbd_dev->lock_dwork, 0); + __rbd_release_lock(rbd_dev); + queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0); } else { __rbd_lock(rbd_dev, cookie); + wake_lock_waiters(rbd_dev, 0); } } @@ -3671,15 +4689,18 @@ static void rbd_reregister_watch(struct work_struct *work) ret = __rbd_register_watch(rbd_dev); if (ret) { rbd_warn(rbd_dev, "failed to reregister watch: %d", ret); - if (ret == -EBLACKLISTED || ret == -ENOENT) { - set_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags); - wake_requests(rbd_dev, true); - } else { + if (ret != -EBLACKLISTED && ret != -ENOENT) { queue_delayed_work(rbd_dev->task_wq, &rbd_dev->watch_dwork, RBD_RETRY_DELAY); + mutex_unlock(&rbd_dev->watch_mutex); + return; } + mutex_unlock(&rbd_dev->watch_mutex); + down_write(&rbd_dev->lock_rwsem); + wake_lock_waiters(rbd_dev, ret); + up_write(&rbd_dev->lock_rwsem); return; } @@ -3742,7 +4763,7 @@ static int rbd_obj_method_sync(struct rbd_device *rbd_dev, ret = ceph_osdc_call(osdc, oid, oloc, RBD_DRV_NAME, method_name, CEPH_OSD_FLAG_READ, req_page, outbound_size, - reply_page, &inbound_size); + &reply_page, &inbound_size); if (!ret) { memcpy(inbound, page_address(reply_page), inbound_size); ret = inbound_size; @@ -3754,54 +4775,6 @@ static int rbd_obj_method_sync(struct rbd_device *rbd_dev, return ret; } -/* - * lock_rwsem must be held for read - */ -static int rbd_wait_state_locked(struct rbd_device *rbd_dev, bool may_acquire) -{ - DEFINE_WAIT(wait); - unsigned long timeout; - int ret = 0; - - if (test_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags)) - return -EBLACKLISTED; - - if (rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED) - return 0; - - if (!may_acquire) { - rbd_warn(rbd_dev, "exclusive lock required"); - return -EROFS; - } - - do { - /* - * Note the use of mod_delayed_work() in rbd_acquire_lock() - * and cancel_delayed_work() in wake_requests(). - */ - dout("%s rbd_dev %p queueing lock_dwork\n", __func__, rbd_dev); - queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0); - prepare_to_wait_exclusive(&rbd_dev->lock_waitq, &wait, - TASK_UNINTERRUPTIBLE); - up_read(&rbd_dev->lock_rwsem); - timeout = schedule_timeout(ceph_timeout_jiffies( - rbd_dev->opts->lock_timeout)); - down_read(&rbd_dev->lock_rwsem); - if (test_bit(RBD_DEV_FLAG_BLACKLISTED, &rbd_dev->flags)) { - ret = -EBLACKLISTED; - break; - } - if (!timeout) { - rbd_warn(rbd_dev, "timed out waiting for lock"); - ret = -ETIMEDOUT; - break; - } - } while (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED); - - finish_wait(&rbd_dev->lock_waitq, &wait); - return ret; -} - static void rbd_queue_workfn(struct work_struct *work) { struct request *rq = blk_mq_rq_from_pdu(work); @@ -3812,7 +4785,6 @@ static void rbd_queue_workfn(struct work_struct *work) u64 length = blk_rq_bytes(rq); enum obj_operation_type op_type; u64 mapping_size; - bool must_be_locked; int result; switch (req_op(rq)) { @@ -3886,21 +4858,10 @@ static void rbd_queue_workfn(struct work_struct *work) goto err_rq; } - must_be_locked = - (rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK) && - (op_type != OBJ_OP_READ || rbd_dev->opts->lock_on_read); - if (must_be_locked) { - down_read(&rbd_dev->lock_rwsem); - result = rbd_wait_state_locked(rbd_dev, - !rbd_dev->opts->exclusive); - if (result) - goto err_unlock; - } - img_request = rbd_img_request_create(rbd_dev, op_type, snapc); if (!img_request) { result = -ENOMEM; - goto err_unlock; + goto err_rq; } img_request->rq = rq; snapc = NULL; /* img_request consumes a ref */ @@ -3910,19 +4871,14 @@ static void rbd_queue_workfn(struct work_struct *work) else result = rbd_img_fill_from_bio(img_request, offset, length, rq->bio); - if (result || !img_request->pending_count) + if (result) goto err_img_request; - rbd_img_request_submit(img_request); - if (must_be_locked) - up_read(&rbd_dev->lock_rwsem); + rbd_img_handle_request(img_request, 0); return; err_img_request: rbd_img_request_put(img_request); -err_unlock: - if (must_be_locked) - up_read(&rbd_dev->lock_rwsem); err_rq: if (result) rbd_warn(rbd_dev, "%s %llx at %llx result %d", @@ -4589,7 +5545,13 @@ static struct rbd_device *__rbd_dev_create(struct rbd_client *rbdc, INIT_WORK(&rbd_dev->released_lock_work, rbd_notify_released_lock); INIT_DELAYED_WORK(&rbd_dev->lock_dwork, rbd_acquire_lock); INIT_WORK(&rbd_dev->unlock_work, rbd_release_lock_work); - init_waitqueue_head(&rbd_dev->lock_waitq); + spin_lock_init(&rbd_dev->lock_lists_lock); + INIT_LIST_HEAD(&rbd_dev->acquiring_list); + INIT_LIST_HEAD(&rbd_dev->running_list); + init_completion(&rbd_dev->acquire_wait); + init_completion(&rbd_dev->releasing_wait); + + spin_lock_init(&rbd_dev->object_map_lock); rbd_dev->dev.bus = &rbd_bus_type; rbd_dev->dev.type = &rbd_device_type; @@ -4772,6 +5734,32 @@ static int rbd_dev_v2_features(struct rbd_device *rbd_dev) &rbd_dev->header.features); } +/* + * These are generic image flags, but since they are used only for + * object map, store them in rbd_dev->object_map_flags. + * + * For the same reason, this function is called only on object map + * (re)load and not on header refresh. + */ +static int rbd_dev_v2_get_flags(struct rbd_device *rbd_dev) +{ + __le64 snapid = cpu_to_le64(rbd_dev->spec->snap_id); + __le64 flags; + int ret; + + ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid, + &rbd_dev->header_oloc, "get_flags", + &snapid, sizeof(snapid), + &flags, sizeof(flags)); + if (ret < 0) + return ret; + if (ret < sizeof(flags)) + return -EBADMSG; + + rbd_dev->object_map_flags = le64_to_cpu(flags); + return 0; +} + struct parent_image_info { u64 pool_id; const char *pool_ns; @@ -4829,7 +5817,7 @@ static int __get_parent_info(struct rbd_device *rbd_dev, ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc, "rbd", "parent_get", CEPH_OSD_FLAG_READ, - req_page, sizeof(u64), reply_page, &reply_len); + req_page, sizeof(u64), &reply_page, &reply_len); if (ret) return ret == -EOPNOTSUPP ? 1 : ret; @@ -4841,7 +5829,7 @@ static int __get_parent_info(struct rbd_device *rbd_dev, ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc, "rbd", "parent_overlap_get", CEPH_OSD_FLAG_READ, - req_page, sizeof(u64), reply_page, &reply_len); + req_page, sizeof(u64), &reply_page, &reply_len); if (ret) return ret; @@ -4872,7 +5860,7 @@ static int __get_parent_info_legacy(struct rbd_device *rbd_dev, ret = ceph_osdc_call(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc, "rbd", "get_parent", CEPH_OSD_FLAG_READ, - req_page, sizeof(u64), reply_page, &reply_len); + req_page, sizeof(u64), &reply_page, &reply_len); if (ret) return ret; @@ -5605,28 +6593,49 @@ static void rbd_dev_image_unlock(struct rbd_device *rbd_dev) { down_write(&rbd_dev->lock_rwsem); if (__rbd_is_lock_owner(rbd_dev)) - rbd_unlock(rbd_dev); + __rbd_release_lock(rbd_dev); up_write(&rbd_dev->lock_rwsem); } +/* + * If the wait is interrupted, an error is returned even if the lock + * was successfully acquired. rbd_dev_image_unlock() will release it + * if needed. + */ static int rbd_add_acquire_lock(struct rbd_device *rbd_dev) { - int ret; + long ret; if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK)) { + if (!rbd_dev->opts->exclusive && !rbd_dev->opts->lock_on_read) + return 0; + rbd_warn(rbd_dev, "exclusive-lock feature is not enabled"); return -EINVAL; } - /* FIXME: "rbd map --exclusive" should be in interruptible */ - down_read(&rbd_dev->lock_rwsem); - ret = rbd_wait_state_locked(rbd_dev, true); - up_read(&rbd_dev->lock_rwsem); + if (rbd_dev->spec->snap_id != CEPH_NOSNAP) + return 0; + + rbd_assert(!rbd_is_lock_owner(rbd_dev)); + queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0); + ret = wait_for_completion_killable_timeout(&rbd_dev->acquire_wait, + ceph_timeout_jiffies(rbd_dev->opts->lock_timeout)); + if (ret > 0) + ret = rbd_dev->acquire_err; + else if (!ret) + ret = -ETIMEDOUT; + if (ret) { - rbd_warn(rbd_dev, "failed to acquire exclusive lock"); - return -EROFS; + rbd_warn(rbd_dev, "failed to acquire exclusive lock: %ld", ret); + return ret; } + /* + * The lock may have been released by now, unless automatic lock + * transitions are disabled. + */ + rbd_assert(!rbd_dev->opts->exclusive || rbd_is_lock_owner(rbd_dev)); return 0; } @@ -5724,6 +6733,8 @@ static void rbd_dev_unprobe(struct rbd_device *rbd_dev) struct rbd_image_header *header; rbd_dev_parent_put(rbd_dev); + rbd_object_map_free(rbd_dev); + rbd_dev_mapping_clear(rbd_dev); /* Free dynamic fields from the header, then zero it out */ @@ -5824,7 +6835,6 @@ out_err: static void rbd_dev_device_release(struct rbd_device *rbd_dev) { clear_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags); - rbd_dev_mapping_clear(rbd_dev); rbd_free_disk(rbd_dev); if (!single_major) unregister_blkdev(rbd_dev->major, rbd_dev->name); @@ -5858,23 +6868,17 @@ static int rbd_dev_device_setup(struct rbd_device *rbd_dev) if (ret) goto err_out_blkdev; - ret = rbd_dev_mapping_set(rbd_dev); - if (ret) - goto err_out_disk; - set_capacity(rbd_dev->disk, rbd_dev->mapping.size / SECTOR_SIZE); set_disk_ro(rbd_dev->disk, rbd_dev->opts->read_only); ret = dev_set_name(&rbd_dev->dev, "%d", rbd_dev->dev_id); if (ret) - goto err_out_mapping; + goto err_out_disk; set_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags); up_write(&rbd_dev->header_rwsem); return 0; -err_out_mapping: - rbd_dev_mapping_clear(rbd_dev); err_out_disk: rbd_free_disk(rbd_dev); err_out_blkdev: @@ -5975,6 +6979,17 @@ static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth) goto err_out_probe; } + ret = rbd_dev_mapping_set(rbd_dev); + if (ret) + goto err_out_probe; + + if (rbd_dev->spec->snap_id != CEPH_NOSNAP && + (rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) { + ret = rbd_object_map_load(rbd_dev); + if (ret) + goto err_out_probe; + } + if (rbd_dev->header.features & RBD_FEATURE_LAYERING) { ret = rbd_dev_v2_parent_info(rbd_dev); if (ret) @@ -6071,11 +7086,9 @@ static ssize_t do_rbd_add(struct bus_type *bus, if (rc) goto err_out_image_probe; - if (rbd_dev->opts->exclusive) { - rc = rbd_add_acquire_lock(rbd_dev); - if (rc) - goto err_out_device_setup; - } + rc = rbd_add_acquire_lock(rbd_dev); + if (rc) + goto err_out_image_lock; /* Everything's ready. Announce the disk to the world. */ @@ -6101,7 +7114,6 @@ out: err_out_image_lock: rbd_dev_image_unlock(rbd_dev); -err_out_device_setup: rbd_dev_device_release(rbd_dev); err_out_image_probe: rbd_dev_image_release(rbd_dev); diff --git a/drivers/block/rbd_types.h b/drivers/block/rbd_types.h index 62ff50d3e7a6..ac98ab6ccd3b 100644 --- a/drivers/block/rbd_types.h +++ b/drivers/block/rbd_types.h @@ -18,6 +18,7 @@ /* For format version 2, rbd image 'foo' consists of objects * rbd_id.foo - id of image * rbd_header.<id> - image metadata + * rbd_object_map.<id> - optional image object map * rbd_data.<id>.0000000000000000 * rbd_data.<id>.0000000000000001 * ... - data @@ -25,6 +26,7 @@ */ #define RBD_HEADER_PREFIX "rbd_header." +#define RBD_OBJECT_MAP_PREFIX "rbd_object_map." #define RBD_ID_PREFIX "rbd_id." #define RBD_V2_DATA_FORMAT "%s.%016llx" @@ -39,6 +41,14 @@ enum rbd_notify_op { RBD_NOTIFY_OP_HEADER_UPDATE = 3, }; +#define OBJECT_NONEXISTENT 0 +#define OBJECT_EXISTS 1 +#define OBJECT_PENDING 2 +#define OBJECT_EXISTS_CLEAN 3 + +#define RBD_FLAG_OBJECT_MAP_INVALID (1ULL << 0) +#define RBD_FLAG_FAST_DIFF_INVALID (1ULL << 1) + /* * For format version 1, rbd image 'foo' consists of objects * foo.rbd - image metadata diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 7376af25f947..801fa1cd0321 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -90,6 +90,17 @@ config COMMON_CLK_SCPI This driver uses SCPI Message Protocol to interact with the firmware providing all the clock controls. +config COMMON_CLK_SI5341 + tristate "Clock driver for SiLabs 5341 and 5340 A/B/C/D devices" + depends on I2C + select REGMAP_I2C + help + This driver supports Silicon Labs Si5341 and Si5340 programmable clock + generators. Not all features of these chips are currently supported + by the driver, in particular it only supports XTAL input. The chip can + be pre-programmed to support other configurations and features not yet + implemented in the driver. + config COMMON_CLK_SI5351 tristate "Clock driver for SiLabs 5351A/B/C" depends on I2C @@ -214,7 +225,7 @@ config CLK_QORIQ config COMMON_CLK_XGENE bool "Clock driver for APM XGene SoC" - default y + default ARCH_XGENE depends on ARM64 || COMPILE_TEST ---help--- Sypport for the APM X-Gene SoC reference, PLL, and device clocks. diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 9ef4305d55e0..0cad76021297 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -49,6 +49,7 @@ obj-$(CONFIG_COMMON_CLK_HI655X) += clk-hi655x.o obj-$(CONFIG_COMMON_CLK_S2MPS11) += clk-s2mps11.o obj-$(CONFIG_COMMON_CLK_SCMI) += clk-scmi.o obj-$(CONFIG_COMMON_CLK_SCPI) += clk-scpi.o +obj-$(CONFIG_COMMON_CLK_SI5341) += clk-si5341.o obj-$(CONFIG_COMMON_CLK_SI5351) += clk-si5351.o obj-$(CONFIG_COMMON_CLK_SI514) += clk-si514.o obj-$(CONFIG_COMMON_CLK_SI544) += clk-si544.o diff --git a/drivers/clk/at91/sckc.c b/drivers/clk/at91/sckc.c index 45526f56f1ba..9bfe9a28294a 100644 --- a/drivers/clk/at91/sckc.c +++ b/drivers/clk/at91/sckc.c @@ -18,14 +18,18 @@ SLOW_CLOCK_FREQ) #define AT91_SCKC_CR 0x00 -#define AT91_SCKC_RCEN (1 << 0) -#define AT91_SCKC_OSC32EN (1 << 1) -#define AT91_SCKC_OSC32BYP (1 << 2) -#define AT91_SCKC_OSCSEL (1 << 3) + +struct clk_slow_bits { + u32 cr_rcen; + u32 cr_osc32en; + u32 cr_osc32byp; + u32 cr_oscsel; +}; struct clk_slow_osc { struct clk_hw hw; void __iomem *sckcr; + const struct clk_slow_bits *bits; unsigned long startup_usec; }; @@ -34,6 +38,7 @@ struct clk_slow_osc { struct clk_sama5d4_slow_osc { struct clk_hw hw; void __iomem *sckcr; + const struct clk_slow_bits *bits; unsigned long startup_usec; bool prepared; }; @@ -43,6 +48,7 @@ struct clk_sama5d4_slow_osc { struct clk_slow_rc_osc { struct clk_hw hw; void __iomem *sckcr; + const struct clk_slow_bits *bits; unsigned long frequency; unsigned long accuracy; unsigned long startup_usec; @@ -53,6 +59,7 @@ struct clk_slow_rc_osc { struct clk_sam9x5_slow { struct clk_hw hw; void __iomem *sckcr; + const struct clk_slow_bits *bits; u8 parent; }; @@ -64,10 +71,10 @@ static int clk_slow_osc_prepare(struct clk_hw *hw) void __iomem *sckcr = osc->sckcr; u32 tmp = readl(sckcr); - if (tmp & (AT91_SCKC_OSC32BYP | AT91_SCKC_OSC32EN)) + if (tmp & (osc->bits->cr_osc32byp | osc->bits->cr_osc32en)) return 0; - writel(tmp | AT91_SCKC_OSC32EN, sckcr); + writel(tmp | osc->bits->cr_osc32en, sckcr); usleep_range(osc->startup_usec, osc->startup_usec + 1); @@ -80,10 +87,10 @@ static void clk_slow_osc_unprepare(struct clk_hw *hw) void __iomem *sckcr = osc->sckcr; u32 tmp = readl(sckcr); - if (tmp & AT91_SCKC_OSC32BYP) + if (tmp & osc->bits->cr_osc32byp) return; - writel(tmp & ~AT91_SCKC_OSC32EN, sckcr); + writel(tmp & ~osc->bits->cr_osc32en, sckcr); } static int clk_slow_osc_is_prepared(struct clk_hw *hw) @@ -92,10 +99,10 @@ static int clk_slow_osc_is_prepared(struct clk_hw *hw) void __iomem *sckcr = osc->sckcr; u32 tmp = readl(sckcr); - if (tmp & AT91_SCKC_OSC32BYP) + if (tmp & osc->bits->cr_osc32byp) return 1; - return !!(tmp & AT91_SCKC_OSC32EN); + return !!(tmp & osc->bits->cr_osc32en); } static const struct clk_ops slow_osc_ops = { @@ -109,7 +116,8 @@ at91_clk_register_slow_osc(void __iomem *sckcr, const char *name, const char *parent_name, unsigned long startup, - bool bypass) + bool bypass, + const struct clk_slow_bits *bits) { struct clk_slow_osc *osc; struct clk_hw *hw; @@ -132,10 +140,11 @@ at91_clk_register_slow_osc(void __iomem *sckcr, osc->hw.init = &init; osc->sckcr = sckcr; osc->startup_usec = startup; + osc->bits = bits; if (bypass) - writel((readl(sckcr) & ~AT91_SCKC_OSC32EN) | AT91_SCKC_OSC32BYP, - sckcr); + writel((readl(sckcr) & ~osc->bits->cr_osc32en) | + osc->bits->cr_osc32byp, sckcr); hw = &osc->hw; ret = clk_hw_register(NULL, &osc->hw); @@ -147,6 +156,14 @@ at91_clk_register_slow_osc(void __iomem *sckcr, return hw; } +static void at91_clk_unregister_slow_osc(struct clk_hw *hw) +{ + struct clk_slow_osc *osc = to_clk_slow_osc(hw); + + clk_hw_unregister(hw); + kfree(osc); +} + static unsigned long clk_slow_rc_osc_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { @@ -168,7 +185,7 @@ static int clk_slow_rc_osc_prepare(struct clk_hw *hw) struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw); void __iomem *sckcr = osc->sckcr; - writel(readl(sckcr) | AT91_SCKC_RCEN, sckcr); + writel(readl(sckcr) | osc->bits->cr_rcen, sckcr); usleep_range(osc->startup_usec, osc->startup_usec + 1); @@ -180,14 +197,14 @@ static void clk_slow_rc_osc_unprepare(struct clk_hw *hw) struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw); void __iomem *sckcr = osc->sckcr; - writel(readl(sckcr) & ~AT91_SCKC_RCEN, sckcr); + writel(readl(sckcr) & ~osc->bits->cr_rcen, sckcr); } static int clk_slow_rc_osc_is_prepared(struct clk_hw *hw) { struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw); - return !!(readl(osc->sckcr) & AT91_SCKC_RCEN); + return !!(readl(osc->sckcr) & osc->bits->cr_rcen); } static const struct clk_ops slow_rc_osc_ops = { @@ -203,7 +220,8 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr, const char *name, unsigned long frequency, unsigned long accuracy, - unsigned long startup) + unsigned long startup, + const struct clk_slow_bits *bits) { struct clk_slow_rc_osc *osc; struct clk_hw *hw; @@ -225,6 +243,7 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr, osc->hw.init = &init; osc->sckcr = sckcr; + osc->bits = bits; osc->frequency = frequency; osc->accuracy = accuracy; osc->startup_usec = startup; @@ -239,6 +258,14 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr, return hw; } +static void at91_clk_unregister_slow_rc_osc(struct clk_hw *hw) +{ + struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw); + + clk_hw_unregister(hw); + kfree(osc); +} + static int clk_sam9x5_slow_set_parent(struct clk_hw *hw, u8 index) { struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw); @@ -250,14 +277,14 @@ static int clk_sam9x5_slow_set_parent(struct clk_hw *hw, u8 index) tmp = readl(sckcr); - if ((!index && !(tmp & AT91_SCKC_OSCSEL)) || - (index && (tmp & AT91_SCKC_OSCSEL))) + if ((!index && !(tmp & slowck->bits->cr_oscsel)) || + (index && (tmp & slowck->bits->cr_oscsel))) return 0; if (index) - tmp |= AT91_SCKC_OSCSEL; + tmp |= slowck->bits->cr_oscsel; else - tmp &= ~AT91_SCKC_OSCSEL; + tmp &= ~slowck->bits->cr_oscsel; writel(tmp, sckcr); @@ -270,7 +297,7 @@ static u8 clk_sam9x5_slow_get_parent(struct clk_hw *hw) { struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw); - return !!(readl(slowck->sckcr) & AT91_SCKC_OSCSEL); + return !!(readl(slowck->sckcr) & slowck->bits->cr_oscsel); } static const struct clk_ops sam9x5_slow_ops = { @@ -282,7 +309,8 @@ static struct clk_hw * __init at91_clk_register_sam9x5_slow(void __iomem *sckcr, const char *name, const char **parent_names, - int num_parents) + int num_parents, + const struct clk_slow_bits *bits) { struct clk_sam9x5_slow *slowck; struct clk_hw *hw; @@ -304,7 +332,8 @@ at91_clk_register_sam9x5_slow(void __iomem *sckcr, slowck->hw.init = &init; slowck->sckcr = sckcr; - slowck->parent = !!(readl(sckcr) & AT91_SCKC_OSCSEL); + slowck->bits = bits; + slowck->parent = !!(readl(sckcr) & slowck->bits->cr_oscsel); hw = &slowck->hw; ret = clk_hw_register(NULL, &slowck->hw); @@ -316,22 +345,33 @@ at91_clk_register_sam9x5_slow(void __iomem *sckcr, return hw; } +static void at91_clk_unregister_sam9x5_slow(struct clk_hw *hw) +{ + struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw); + + clk_hw_unregister(hw); + kfree(slowck); +} + static void __init at91sam9x5_sckc_register(struct device_node *np, - unsigned int rc_osc_startup_us) + unsigned int rc_osc_startup_us, + const struct clk_slow_bits *bits) { const char *parent_names[2] = { "slow_rc_osc", "slow_osc" }; void __iomem *regbase = of_iomap(np, 0); struct device_node *child = NULL; const char *xtal_name; - struct clk_hw *hw; + struct clk_hw *slow_rc, *slow_osc, *slowck; bool bypass; + int ret; if (!regbase) return; - hw = at91_clk_register_slow_rc_osc(regbase, parent_names[0], 32768, - 50000000, rc_osc_startup_us); - if (IS_ERR(hw)) + slow_rc = at91_clk_register_slow_rc_osc(regbase, parent_names[0], + 32768, 50000000, + rc_osc_startup_us, bits); + if (IS_ERR(slow_rc)) return; xtal_name = of_clk_get_parent_name(np, 0); @@ -339,7 +379,7 @@ static void __init at91sam9x5_sckc_register(struct device_node *np, /* DT backward compatibility */ child = of_get_compatible_child(np, "atmel,at91sam9x5-clk-slow-osc"); if (!child) - return; + goto unregister_slow_rc; xtal_name = of_clk_get_parent_name(child, 0); bypass = of_property_read_bool(child, "atmel,osc-bypass"); @@ -350,38 +390,133 @@ static void __init at91sam9x5_sckc_register(struct device_node *np, } if (!xtal_name) - return; - - hw = at91_clk_register_slow_osc(regbase, parent_names[1], xtal_name, - 1200000, bypass); - if (IS_ERR(hw)) - return; + goto unregister_slow_rc; - hw = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names, 2); - if (IS_ERR(hw)) - return; + slow_osc = at91_clk_register_slow_osc(regbase, parent_names[1], + xtal_name, 1200000, bypass, bits); + if (IS_ERR(slow_osc)) + goto unregister_slow_rc; - of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); + slowck = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names, + 2, bits); + if (IS_ERR(slowck)) + goto unregister_slow_osc; /* DT backward compatibility */ if (child) - of_clk_add_hw_provider(child, of_clk_hw_simple_get, hw); + ret = of_clk_add_hw_provider(child, of_clk_hw_simple_get, + slowck); + else + ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, slowck); + + if (WARN_ON(ret)) + goto unregister_slowck; + + return; + +unregister_slowck: + at91_clk_unregister_sam9x5_slow(slowck); +unregister_slow_osc: + at91_clk_unregister_slow_osc(slow_osc); +unregister_slow_rc: + at91_clk_unregister_slow_rc_osc(slow_rc); } +static const struct clk_slow_bits at91sam9x5_bits = { + .cr_rcen = BIT(0), + .cr_osc32en = BIT(1), + .cr_osc32byp = BIT(2), + .cr_oscsel = BIT(3), +}; + static void __init of_at91sam9x5_sckc_setup(struct device_node *np) { - at91sam9x5_sckc_register(np, 75); + at91sam9x5_sckc_register(np, 75, &at91sam9x5_bits); } CLK_OF_DECLARE(at91sam9x5_clk_sckc, "atmel,at91sam9x5-sckc", of_at91sam9x5_sckc_setup); static void __init of_sama5d3_sckc_setup(struct device_node *np) { - at91sam9x5_sckc_register(np, 500); + at91sam9x5_sckc_register(np, 500, &at91sam9x5_bits); } CLK_OF_DECLARE(sama5d3_clk_sckc, "atmel,sama5d3-sckc", of_sama5d3_sckc_setup); +static const struct clk_slow_bits at91sam9x60_bits = { + .cr_osc32en = BIT(1), + .cr_osc32byp = BIT(2), + .cr_oscsel = BIT(24), +}; + +static void __init of_sam9x60_sckc_setup(struct device_node *np) +{ + void __iomem *regbase = of_iomap(np, 0); + struct clk_hw_onecell_data *clk_data; + struct clk_hw *slow_rc, *slow_osc; + const char *xtal_name; + const char *parent_names[2] = { "slow_rc_osc", "slow_osc" }; + bool bypass; + int ret; + + if (!regbase) + return; + + slow_rc = clk_hw_register_fixed_rate(NULL, parent_names[0], NULL, 0, + 32768); + if (IS_ERR(slow_rc)) + return; + + xtal_name = of_clk_get_parent_name(np, 0); + if (!xtal_name) + goto unregister_slow_rc; + + bypass = of_property_read_bool(np, "atmel,osc-bypass"); + slow_osc = at91_clk_register_slow_osc(regbase, parent_names[1], + xtal_name, 5000000, bypass, + &at91sam9x60_bits); + if (IS_ERR(slow_osc)) + goto unregister_slow_rc; + + clk_data = kzalloc(sizeof(*clk_data) + (2 * sizeof(struct clk_hw *)), + GFP_KERNEL); + if (!clk_data) + goto unregister_slow_osc; + + /* MD_SLCK and TD_SLCK. */ + clk_data->num = 2; + clk_data->hws[0] = clk_hw_register_fixed_rate(NULL, "md_slck", + parent_names[0], + 0, 32768); + if (IS_ERR(clk_data->hws[0])) + goto clk_data_free; + + clk_data->hws[1] = at91_clk_register_sam9x5_slow(regbase, "td_slck", + parent_names, 2, + &at91sam9x60_bits); + if (IS_ERR(clk_data->hws[1])) + goto unregister_md_slck; + + ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); + if (WARN_ON(ret)) + goto unregister_td_slck; + + return; + +unregister_td_slck: + at91_clk_unregister_sam9x5_slow(clk_data->hws[1]); +unregister_md_slck: + clk_hw_unregister(clk_data->hws[0]); +clk_data_free: + kfree(clk_data); +unregister_slow_osc: + at91_clk_unregister_slow_osc(slow_osc); +unregister_slow_rc: + clk_hw_unregister(slow_rc); +} +CLK_OF_DECLARE(sam9x60_clk_sckc, "microchip,sam9x60-sckc", + of_sam9x60_sckc_setup); + static int clk_sama5d4_slow_osc_prepare(struct clk_hw *hw) { struct clk_sama5d4_slow_osc *osc = to_clk_sama5d4_slow_osc(hw); @@ -393,7 +528,7 @@ static int clk_sama5d4_slow_osc_prepare(struct clk_hw *hw) * Assume that if it has already been selected (for example by the * bootloader), enough time has aready passed. */ - if ((readl(osc->sckcr) & AT91_SCKC_OSCSEL)) { + if ((readl(osc->sckcr) & osc->bits->cr_oscsel)) { osc->prepared = true; return 0; } @@ -416,33 +551,35 @@ static const struct clk_ops sama5d4_slow_osc_ops = { .is_prepared = clk_sama5d4_slow_osc_is_prepared, }; +static const struct clk_slow_bits at91sama5d4_bits = { + .cr_oscsel = BIT(3), +}; + static void __init of_sama5d4_sckc_setup(struct device_node *np) { void __iomem *regbase = of_iomap(np, 0); - struct clk_hw *hw; + struct clk_hw *slow_rc, *slowck; struct clk_sama5d4_slow_osc *osc; struct clk_init_data init; const char *xtal_name; const char *parent_names[2] = { "slow_rc_osc", "slow_osc" }; - bool bypass; int ret; if (!regbase) return; - hw = clk_hw_register_fixed_rate_with_accuracy(NULL, parent_names[0], - NULL, 0, 32768, - 250000000); - if (IS_ERR(hw)) + slow_rc = clk_hw_register_fixed_rate_with_accuracy(NULL, + parent_names[0], + NULL, 0, 32768, + 250000000); + if (IS_ERR(slow_rc)) return; xtal_name = of_clk_get_parent_name(np, 0); - bypass = of_property_read_bool(np, "atmel,osc-bypass"); - osc = kzalloc(sizeof(*osc), GFP_KERNEL); if (!osc) - return; + goto unregister_slow_rc; init.name = parent_names[1]; init.ops = &sama5d4_slow_osc_ops; @@ -453,22 +590,32 @@ static void __init of_sama5d4_sckc_setup(struct device_node *np) osc->hw.init = &init; osc->sckcr = regbase; osc->startup_usec = 1200000; + osc->bits = &at91sama5d4_bits; - if (bypass) - writel((readl(regbase) | AT91_SCKC_OSC32BYP), regbase); - - hw = &osc->hw; ret = clk_hw_register(NULL, &osc->hw); - if (ret) { - kfree(osc); - return; - } - - hw = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names, 2); - if (IS_ERR(hw)) - return; - - of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); + if (ret) + goto free_slow_osc_data; + + slowck = at91_clk_register_sam9x5_slow(regbase, "slowck", + parent_names, 2, + &at91sama5d4_bits); + if (IS_ERR(slowck)) + goto unregister_slow_osc; + + ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, slowck); + if (WARN_ON(ret)) + goto unregister_slowck; + + return; + +unregister_slowck: + at91_clk_unregister_sam9x5_slow(slowck); +unregister_slow_osc: + clk_hw_unregister(&osc->hw); +free_slow_osc_data: + kfree(osc); +unregister_slow_rc: + clk_hw_unregister(slow_rc); } CLK_OF_DECLARE(sama5d4_clk_sckc, "atmel,sama5d4-sckc", of_sama5d4_sckc_setup); diff --git a/drivers/clk/bcm/Kconfig b/drivers/clk/bcm/Kconfig index 29ee7b776cd4..8c83977a7dc4 100644 --- a/drivers/clk/bcm/Kconfig +++ b/drivers/clk/bcm/Kconfig @@ -1,4 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only +config CLK_BCM2835 + bool "Broadcom BCM2835 clock support" + depends on ARCH_BCM2835 || ARCH_BRCMSTB || COMPILE_TEST + depends on COMMON_CLK + default ARCH_BCM2835 || ARCH_BRCMSTB + help + Enable common clock framework support for Broadcom BCM2835 + SoCs. + config CLK_BCM_63XX bool "Broadcom BCM63xx clock support" depends on ARCH_BCM_63XX || COMPILE_TEST @@ -8,6 +17,14 @@ config CLK_BCM_63XX Enable common clock framework support for Broadcom BCM63xx DSL SoCs based on the ARM architecture +config CLK_BCM_63XX_GATE + bool "Broadcom BCM63xx gated clock support" + depends on BMIPS_GENERIC || COMPILE_TEST + default BMIPS_GENERIC + help + Enable common clock framework support for Broadcom BCM63xx DSL SoCs + based on the MIPS architecture + config CLK_BCM_KONA bool "Broadcom Kona CCU clock support" depends on ARCH_BCM_MOBILE || COMPILE_TEST @@ -64,3 +81,10 @@ config CLK_BCM_SR default ARCH_BCM_IPROC help Enable common clock framework support for the Broadcom Stingray SoC + +config CLK_RASPBERRYPI + tristate "Raspberry Pi firmware based clock support" + depends on RASPBERRYPI_FIRMWARE || (COMPILE_TEST && !RASPBERRYPI_FIRMWARE) + help + Enable common clock framework support for Raspberry Pi's firmware + dependent clocks diff --git a/drivers/clk/bcm/Makefile b/drivers/clk/bcm/Makefile index 002661d39128..0070ddf6cdd2 100644 --- a/drivers/clk/bcm/Makefile +++ b/drivers/clk/bcm/Makefile @@ -1,12 +1,14 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CLK_BCM_63XX) += clk-bcm63xx.o +obj-$(CONFIG_CLK_BCM_63XX_GATE) += clk-bcm63xx-gate.o obj-$(CONFIG_CLK_BCM_KONA) += clk-kona.o obj-$(CONFIG_CLK_BCM_KONA) += clk-kona-setup.o obj-$(CONFIG_CLK_BCM_KONA) += clk-bcm281xx.o obj-$(CONFIG_CLK_BCM_KONA) += clk-bcm21664.o obj-$(CONFIG_COMMON_CLK_IPROC) += clk-iproc-armpll.o clk-iproc-pll.o clk-iproc-asiu.o -obj-$(CONFIG_ARCH_BCM2835) += clk-bcm2835.o -obj-$(CONFIG_ARCH_BCM2835) += clk-bcm2835-aux.o +obj-$(CONFIG_CLK_BCM2835) += clk-bcm2835.o +obj-$(CONFIG_CLK_BCM2835) += clk-bcm2835-aux.o +obj-$(CONFIG_CLK_RASPBERRYPI) += clk-raspberrypi.o obj-$(CONFIG_ARCH_BCM_53573) += clk-bcm53573-ilp.o obj-$(CONFIG_CLK_BCM_CYGNUS) += clk-cygnus.o obj-$(CONFIG_CLK_BCM_HR2) += clk-hr2.o diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c index 770bb01f523e..867ae3c20041 100644 --- a/drivers/clk/bcm/clk-bcm2835.c +++ b/drivers/clk/bcm/clk-bcm2835.c @@ -1651,30 +1651,10 @@ static const struct bcm2835_clk_desc clk_desc_array[] = { .fixed_divider = 1, .flags = CLK_SET_RATE_PARENT), - /* PLLB is used for the ARM's clock. */ - [BCM2835_PLLB] = REGISTER_PLL( - .name = "pllb", - .cm_ctrl_reg = CM_PLLB, - .a2w_ctrl_reg = A2W_PLLB_CTRL, - .frac_reg = A2W_PLLB_FRAC, - .ana_reg_base = A2W_PLLB_ANA0, - .reference_enable_mask = A2W_XOSC_CTRL_PLLB_ENABLE, - .lock_mask = CM_LOCK_FLOCKB, - - .ana = &bcm2835_ana_default, - - .min_rate = 600000000u, - .max_rate = 3000000000u, - .max_fb_rate = BCM2835_MAX_FB_RATE), - [BCM2835_PLLB_ARM] = REGISTER_PLL_DIV( - .name = "pllb_arm", - .source_pll = "pllb", - .cm_reg = CM_PLLB, - .a2w_reg = A2W_PLLB_ARM, - .load_mask = CM_PLLB_LOADARM, - .hold_mask = CM_PLLB_HOLDARM, - .fixed_divider = 1, - .flags = CLK_SET_RATE_PARENT), + /* + * PLLB is used for the ARM's clock. Controlled by firmware, see + * clk-raspberrypi.c. + */ /* * PLLC is the core PLL, used to drive the core VPU clock. diff --git a/drivers/clk/bcm/clk-bcm63xx-gate.c b/drivers/clk/bcm/clk-bcm63xx-gate.c new file mode 100644 index 000000000000..9e1dcd43258c --- /dev/null +++ b/drivers/clk/bcm/clk-bcm63xx-gate.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/clk-provider.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +struct clk_bcm63xx_table_entry { + const char * const name; + u8 bit; + unsigned long flags; +}; + +struct clk_bcm63xx_hw { + void __iomem *regs; + spinlock_t lock; + + struct clk_hw_onecell_data data; +}; + +static const struct clk_bcm63xx_table_entry bcm3368_clocks[] = { + { .name = "mac", .bit = 3, }, + { .name = "tc", .bit = 5, }, + { .name = "us_top", .bit = 6, }, + { .name = "ds_top", .bit = 7, }, + { .name = "acm", .bit = 8, }, + { .name = "spi", .bit = 9, }, + { .name = "usbs", .bit = 10, }, + { .name = "bmu", .bit = 11, }, + { .name = "pcm", .bit = 12, }, + { .name = "ntp", .bit = 13, }, + { .name = "acp_b", .bit = 14, }, + { .name = "acp_a", .bit = 15, }, + { .name = "emusb", .bit = 17, }, + { .name = "enet0", .bit = 18, }, + { .name = "enet1", .bit = 19, }, + { .name = "usbsu", .bit = 20, }, + { .name = "ephy", .bit = 21, }, + { }, +}; + +static const struct clk_bcm63xx_table_entry bcm6328_clocks[] = { + { .name = "phy_mips", .bit = 0, }, + { .name = "adsl_qproc", .bit = 1, }, + { .name = "adsl_afe", .bit = 2, }, + { .name = "adsl", .bit = 3, }, + { .name = "mips", .bit = 4, .flags = CLK_IS_CRITICAL, }, + { .name = "sar", .bit = 5, }, + { .name = "pcm", .bit = 6, }, + { .name = "usbd", .bit = 7, }, + { .name = "usbh", .bit = 8, }, + { .name = "hsspi", .bit = 9, }, + { .name = "pcie", .bit = 10, }, + { .name = "robosw", .bit = 11, }, + { }, +}; + +static const struct clk_bcm63xx_table_entry bcm6358_clocks[] = { + { .name = "enet", .bit = 4, }, + { .name = "adslphy", .bit = 5, }, + { .name = "pcm", .bit = 8, }, + { .name = "spi", .bit = 9, }, + { .name = "usbs", .bit = 10, }, + { .name = "sar", .bit = 11, }, + { .name = "emusb", .bit = 17, }, + { .name = "enet0", .bit = 18, }, + { .name = "enet1", .bit = 19, }, + { .name = "usbsu", .bit = 20, }, + { .name = "ephy", .bit = 21, }, + { }, +}; + +static const struct clk_bcm63xx_table_entry bcm6362_clocks[] = { + { .name = "adsl_qproc", .bit = 1, }, + { .name = "adsl_afe", .bit = 2, }, + { .name = "adsl", .bit = 3, }, + { .name = "mips", .bit = 4, .flags = CLK_IS_CRITICAL, }, + { .name = "wlan_ocp", .bit = 5, }, + { .name = "swpkt_usb", .bit = 7, }, + { .name = "swpkt_sar", .bit = 8, }, + { .name = "sar", .bit = 9, }, + { .name = "robosw", .bit = 10, }, + { .name = "pcm", .bit = 11, }, + { .name = "usbd", .bit = 12, }, + { .name = "usbh", .bit = 13, }, + { .name = "ipsec", .bit = 14, }, + { .name = "spi", .bit = 15, }, + { .name = "hsspi", .bit = 16, }, + { .name = "pcie", .bit = 17, }, + { .name = "fap", .bit = 18, }, + { .name = "phymips", .bit = 19, }, + { .name = "nand", .bit = 20, }, + { }, +}; + +static const struct clk_bcm63xx_table_entry bcm6368_clocks[] = { + { .name = "vdsl_qproc", .bit = 2, }, + { .name = "vdsl_afe", .bit = 3, }, + { .name = "vdsl_bonding", .bit = 4, }, + { .name = "vdsl", .bit = 5, }, + { .name = "phymips", .bit = 6, }, + { .name = "swpkt_usb", .bit = 7, }, + { .name = "swpkt_sar", .bit = 8, }, + { .name = "spi", .bit = 9, }, + { .name = "usbd", .bit = 10, }, + { .name = "sar", .bit = 11, }, + { .name = "robosw", .bit = 12, }, + { .name = "utopia", .bit = 13, }, + { .name = "pcm", .bit = 14, }, + { .name = "usbh", .bit = 15, }, + { .name = "disable_gless", .bit = 16, }, + { .name = "nand", .bit = 17, }, + { .name = "ipsec", .bit = 18, }, + { }, +}; + +static const struct clk_bcm63xx_table_entry bcm63268_clocks[] = { + { .name = "disable_gless", .bit = 0, }, + { .name = "vdsl_qproc", .bit = 1, }, + { .name = "vdsl_afe", .bit = 2, }, + { .name = "vdsl", .bit = 3, }, + { .name = "mips", .bit = 4, .flags = CLK_IS_CRITICAL, }, + { .name = "wlan_ocp", .bit = 5, }, + { .name = "dect", .bit = 6, }, + { .name = "fap0", .bit = 7, }, + { .name = "fap1", .bit = 8, }, + { .name = "sar", .bit = 9, }, + { .name = "robosw", .bit = 10, }, + { .name = "pcm", .bit = 11, }, + { .name = "usbd", .bit = 12, }, + { .name = "usbh", .bit = 13, }, + { .name = "ipsec", .bit = 14, }, + { .name = "spi", .bit = 15, }, + { .name = "hsspi", .bit = 16, }, + { .name = "pcie", .bit = 17, }, + { .name = "phymips", .bit = 18, }, + { .name = "gmac", .bit = 19, }, + { .name = "nand", .bit = 20, }, + { .name = "tbus", .bit = 27, }, + { .name = "robosw250", .bit = 31, }, + { }, +}; + +static int clk_bcm63xx_probe(struct platform_device *pdev) +{ + const struct clk_bcm63xx_table_entry *entry, *table; + struct clk_bcm63xx_hw *hw; + struct resource *r; + u8 maxbit = 0; + int i, ret; + + table = of_device_get_match_data(&pdev->dev); + if (!table) + return -EINVAL; + + for (entry = table; entry->name; entry++) + maxbit = max_t(u8, maxbit, entry->bit); + + hw = devm_kzalloc(&pdev->dev, struct_size(hw, data.hws, maxbit), + GFP_KERNEL); + if (!hw) + return -ENOMEM; + + platform_set_drvdata(pdev, hw); + + spin_lock_init(&hw->lock); + + hw->data.num = maxbit; + for (i = 0; i < maxbit; i++) + hw->data.hws[i] = ERR_PTR(-ENODEV); + + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + hw->regs = devm_ioremap_resource(&pdev->dev, r); + if (IS_ERR(hw->regs)) + return PTR_ERR(hw->regs); + + for (entry = table; entry->name; entry++) { + struct clk_hw *clk; + + clk = clk_hw_register_gate(&pdev->dev, entry->name, NULL, + entry->flags, hw->regs, entry->bit, + CLK_GATE_BIG_ENDIAN, &hw->lock); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + goto out_err; + } + + hw->data.hws[entry->bit] = clk; + } + + ret = of_clk_add_hw_provider(pdev->dev.of_node, of_clk_hw_onecell_get, + &hw->data); + if (!ret) + return 0; +out_err: + for (i = 0; i < hw->data.num; i++) { + if (!IS_ERR(hw->data.hws[i])) + clk_hw_unregister_gate(hw->data.hws[i]); + } + + return ret; +} + +static int clk_bcm63xx_remove(struct platform_device *pdev) +{ + struct clk_bcm63xx_hw *hw = platform_get_drvdata(pdev); + int i; + + of_clk_del_provider(pdev->dev.of_node); + + for (i = 0; i < hw->data.num; i++) { + if (!IS_ERR(hw->data.hws[i])) + clk_hw_unregister_gate(hw->data.hws[i]); + } + + return 0; +} + +static const struct of_device_id clk_bcm63xx_dt_ids[] = { + { .compatible = "brcm,bcm3368-clocks", .data = &bcm3368_clocks, }, + { .compatible = "brcm,bcm6328-clocks", .data = &bcm6328_clocks, }, + { .compatible = "brcm,bcm6358-clocks", .data = &bcm6358_clocks, }, + { .compatible = "brcm,bcm6362-clocks", .data = &bcm6362_clocks, }, + { .compatible = "brcm,bcm6368-clocks", .data = &bcm6368_clocks, }, + { .compatible = "brcm,bcm63268-clocks", .data = &bcm63268_clocks, }, + { } +}; + +static struct platform_driver clk_bcm63xx = { + .probe = clk_bcm63xx_probe, + .remove = clk_bcm63xx_remove, + .driver = { + .name = "bcm63xx-clock", + .of_match_table = clk_bcm63xx_dt_ids, + }, +}; +builtin_platform_driver(clk_bcm63xx); diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c new file mode 100644 index 000000000000..1654fd0eedc9 --- /dev/null +++ b/drivers/clk/bcm/clk-raspberrypi.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Raspberry Pi driver for firmware controlled clocks + * + * Even though clk-bcm2835 provides an interface to the hardware registers for + * the system clocks we've had to factor out 'pllb' as the firmware 'owns' it. + * We're not allowed to change it directly as we might race with the + * over-temperature and under-voltage protections provided by the firmware. + * + * Copyright (C) 2019 Nicolas Saenz Julienne <nsaenzjulienne@suse.de> + */ + +#include <linux/clkdev.h> +#include <linux/clk-provider.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +#include <soc/bcm2835/raspberrypi-firmware.h> + +#define RPI_FIRMWARE_ARM_CLK_ID 0x00000003 + +#define RPI_FIRMWARE_STATE_ENABLE_BIT BIT(0) +#define RPI_FIRMWARE_STATE_WAIT_BIT BIT(1) + +/* + * Even though the firmware interface alters 'pllb' the frequencies are + * provided as per 'pllb_arm'. We need to scale before passing them trough. + */ +#define RPI_FIRMWARE_PLLB_ARM_DIV_RATE 2 + +#define A2W_PLL_FRAC_BITS 20 + +struct raspberrypi_clk { + struct device *dev; + struct rpi_firmware *firmware; + struct platform_device *cpufreq; + + unsigned long min_rate; + unsigned long max_rate; + + struct clk_hw pllb; + struct clk_hw *pllb_arm; + struct clk_lookup *pllb_arm_lookup; +}; + +/* + * Structure of the message passed to Raspberry Pi's firmware in order to + * change clock rates. The 'disable_turbo' option is only available to the ARM + * clock (pllb) which we enable by default as turbo mode will alter multiple + * clocks at once. + * + * Even though we're able to access the clock registers directly we're bound to + * use the firmware interface as the firmware ultimately takes care of + * mitigating overheating/undervoltage situations and we would be changing + * frequencies behind his back. + * + * For more information on the firmware interface check: + * https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface + */ +struct raspberrypi_firmware_prop { + __le32 id; + __le32 val; + __le32 disable_turbo; +} __packed; + +static int raspberrypi_clock_property(struct rpi_firmware *firmware, u32 tag, + u32 clk, u32 *val) +{ + struct raspberrypi_firmware_prop msg = { + .id = cpu_to_le32(clk), + .val = cpu_to_le32(*val), + .disable_turbo = cpu_to_le32(1), + }; + int ret; + + ret = rpi_firmware_property(firmware, tag, &msg, sizeof(msg)); + if (ret) + return ret; + + *val = le32_to_cpu(msg.val); + + return 0; +} + +static int raspberrypi_fw_pll_is_on(struct clk_hw *hw) +{ + struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk, + pllb); + u32 val = 0; + int ret; + + ret = raspberrypi_clock_property(rpi->firmware, + RPI_FIRMWARE_GET_CLOCK_STATE, + RPI_FIRMWARE_ARM_CLK_ID, &val); + if (ret) + return 0; + + return !!(val & RPI_FIRMWARE_STATE_ENABLE_BIT); +} + + +static unsigned long raspberrypi_fw_pll_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk, + pllb); + u32 val = 0; + int ret; + + ret = raspberrypi_clock_property(rpi->firmware, + RPI_FIRMWARE_GET_CLOCK_RATE, + RPI_FIRMWARE_ARM_CLK_ID, + &val); + if (ret) + return ret; + + return val * RPI_FIRMWARE_PLLB_ARM_DIV_RATE; +} + +static int raspberrypi_fw_pll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk, + pllb); + u32 new_rate = rate / RPI_FIRMWARE_PLLB_ARM_DIV_RATE; + int ret; + + ret = raspberrypi_clock_property(rpi->firmware, + RPI_FIRMWARE_SET_CLOCK_RATE, + RPI_FIRMWARE_ARM_CLK_ID, + &new_rate); + if (ret) + dev_err_ratelimited(rpi->dev, "Failed to change %s frequency: %d", + clk_hw_get_name(hw), ret); + + return ret; +} + +/* + * Sadly there is no firmware rate rounding interface. We borrowed it from + * clk-bcm2835. + */ +static int raspberrypi_pll_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct raspberrypi_clk *rpi = container_of(hw, struct raspberrypi_clk, + pllb); + u64 div, final_rate; + u32 ndiv, fdiv; + + /* We can't use req->rate directly as it would overflow */ + final_rate = clamp(req->rate, rpi->min_rate, rpi->max_rate); + + div = (u64)final_rate << A2W_PLL_FRAC_BITS; + do_div(div, req->best_parent_rate); + + ndiv = div >> A2W_PLL_FRAC_BITS; + fdiv = div & ((1 << A2W_PLL_FRAC_BITS) - 1); + + final_rate = ((u64)req->best_parent_rate * + ((ndiv << A2W_PLL_FRAC_BITS) + fdiv)); + + req->rate = final_rate >> A2W_PLL_FRAC_BITS; + + return 0; +} + +static const struct clk_ops raspberrypi_firmware_pll_clk_ops = { + .is_prepared = raspberrypi_fw_pll_is_on, + .recalc_rate = raspberrypi_fw_pll_get_rate, + .set_rate = raspberrypi_fw_pll_set_rate, + .determine_rate = raspberrypi_pll_determine_rate, +}; + +static int raspberrypi_register_pllb(struct raspberrypi_clk *rpi) +{ + u32 min_rate = 0, max_rate = 0; + struct clk_init_data init; + int ret; + + memset(&init, 0, sizeof(init)); + + /* All of the PLLs derive from the external oscillator. */ + init.parent_names = (const char *[]){ "osc" }; + init.num_parents = 1; + init.name = "pllb"; + init.ops = &raspberrypi_firmware_pll_clk_ops; + init.flags = CLK_GET_RATE_NOCACHE | CLK_IGNORE_UNUSED; + + /* Get min & max rates set by the firmware */ + ret = raspberrypi_clock_property(rpi->firmware, + RPI_FIRMWARE_GET_MIN_CLOCK_RATE, + RPI_FIRMWARE_ARM_CLK_ID, + &min_rate); + if (ret) { + dev_err(rpi->dev, "Failed to get %s min freq: %d\n", + init.name, ret); + return ret; + } + + ret = raspberrypi_clock_property(rpi->firmware, + RPI_FIRMWARE_GET_MAX_CLOCK_RATE, + RPI_FIRMWARE_ARM_CLK_ID, + &max_rate); + if (ret) { + dev_err(rpi->dev, "Failed to get %s max freq: %d\n", + init.name, ret); + return ret; + } + + if (!min_rate || !max_rate) { + dev_err(rpi->dev, "Unexpected frequency range: min %u, max %u\n", + min_rate, max_rate); + return -EINVAL; + } + + dev_info(rpi->dev, "CPU frequency range: min %u, max %u\n", + min_rate, max_rate); + + rpi->min_rate = min_rate * RPI_FIRMWARE_PLLB_ARM_DIV_RATE; + rpi->max_rate = max_rate * RPI_FIRMWARE_PLLB_ARM_DIV_RATE; + + rpi->pllb.init = &init; + + return devm_clk_hw_register(rpi->dev, &rpi->pllb); +} + +static int raspberrypi_register_pllb_arm(struct raspberrypi_clk *rpi) +{ + rpi->pllb_arm = clk_hw_register_fixed_factor(rpi->dev, + "pllb_arm", "pllb", + CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + 1, 2); + if (IS_ERR(rpi->pllb_arm)) { + dev_err(rpi->dev, "Failed to initialize pllb_arm\n"); + return PTR_ERR(rpi->pllb_arm); + } + + rpi->pllb_arm_lookup = clkdev_hw_create(rpi->pllb_arm, NULL, "cpu0"); + if (!rpi->pllb_arm_lookup) { + dev_err(rpi->dev, "Failed to initialize pllb_arm_lookup\n"); + clk_hw_unregister_fixed_factor(rpi->pllb_arm); + return -ENOMEM; + } + + return 0; +} + +static int raspberrypi_clk_probe(struct platform_device *pdev) +{ + struct device_node *firmware_node; + struct device *dev = &pdev->dev; + struct rpi_firmware *firmware; + struct raspberrypi_clk *rpi; + int ret; + + firmware_node = of_find_compatible_node(NULL, NULL, + "raspberrypi,bcm2835-firmware"); + if (!firmware_node) { + dev_err(dev, "Missing firmware node\n"); + return -ENOENT; + } + + firmware = rpi_firmware_get(firmware_node); + of_node_put(firmware_node); + if (!firmware) + return -EPROBE_DEFER; + + rpi = devm_kzalloc(dev, sizeof(*rpi), GFP_KERNEL); + if (!rpi) + return -ENOMEM; + + rpi->dev = dev; + rpi->firmware = firmware; + platform_set_drvdata(pdev, rpi); + + ret = raspberrypi_register_pllb(rpi); + if (ret) { + dev_err(dev, "Failed to initialize pllb, %d\n", ret); + return ret; + } + + ret = raspberrypi_register_pllb_arm(rpi); + if (ret) + return ret; + + rpi->cpufreq = platform_device_register_data(dev, "raspberrypi-cpufreq", + -1, NULL, 0); + + return 0; +} + +static int raspberrypi_clk_remove(struct platform_device *pdev) +{ + struct raspberrypi_clk *rpi = platform_get_drvdata(pdev); + + platform_device_unregister(rpi->cpufreq); + + return 0; +} + +static struct platform_driver raspberrypi_clk_driver = { + .driver = { + .name = "raspberrypi-clk", + }, + .probe = raspberrypi_clk_probe, + .remove = raspberrypi_clk_remove, +}; +module_platform_driver(raspberrypi_clk_driver); + +MODULE_AUTHOR("Nicolas Saenz Julienne <nsaenzjulienne@suse.de>"); +MODULE_DESCRIPTION("Raspberry Pi firmware clock driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:raspberrypi-clk"); diff --git a/drivers/clk/clk-bulk.c b/drivers/clk/clk-bulk.c index 06499568cf07..524bf9a53098 100644 --- a/drivers/clk/clk-bulk.c +++ b/drivers/clk/clk-bulk.c @@ -75,8 +75,8 @@ void clk_bulk_put(int num_clks, struct clk_bulk_data *clks) } EXPORT_SYMBOL_GPL(clk_bulk_put); -int __must_check clk_bulk_get(struct device *dev, int num_clks, - struct clk_bulk_data *clks) +static int __clk_bulk_get(struct device *dev, int num_clks, + struct clk_bulk_data *clks, bool optional) { int ret; int i; @@ -88,10 +88,14 @@ int __must_check clk_bulk_get(struct device *dev, int num_clks, clks[i].clk = clk_get(dev, clks[i].id); if (IS_ERR(clks[i].clk)) { ret = PTR_ERR(clks[i].clk); + clks[i].clk = NULL; + + if (ret == -ENOENT && optional) + continue; + if (ret != -EPROBE_DEFER) dev_err(dev, "Failed to get clk '%s': %d\n", clks[i].id, ret); - clks[i].clk = NULL; goto err; } } @@ -103,8 +107,21 @@ err: return ret; } + +int __must_check clk_bulk_get(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __clk_bulk_get(dev, num_clks, clks, false); +} EXPORT_SYMBOL(clk_bulk_get); +int __must_check clk_bulk_get_optional(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __clk_bulk_get(dev, num_clks, clks, true); +} +EXPORT_SYMBOL_GPL(clk_bulk_get_optional); + void clk_bulk_put_all(int num_clks, struct clk_bulk_data *clks) { if (IS_ERR_OR_NULL(clks)) diff --git a/drivers/clk/clk-cdce706.c b/drivers/clk/clk-cdce706.c index 0443dfc82794..239102e37e2f 100644 --- a/drivers/clk/clk-cdce706.c +++ b/drivers/clk/clk-cdce706.c @@ -630,7 +630,7 @@ of_clk_cdce_get(struct of_phandle_args *clkspec, void *data) static int cdce706_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; struct cdce706_dev_data *cdce; int ret; diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c index daa1fc8fba53..be160764911b 100644 --- a/drivers/clk/clk-devres.c +++ b/drivers/clk/clk-devres.c @@ -52,8 +52,8 @@ static void devm_clk_bulk_release(struct device *dev, void *res) clk_bulk_put(devres->num_clks, devres->clks); } -int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, - struct clk_bulk_data *clks) +static int __devm_clk_bulk_get(struct device *dev, int num_clks, + struct clk_bulk_data *clks, bool optional) { struct clk_bulk_devres *devres; int ret; @@ -63,7 +63,10 @@ int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, if (!devres) return -ENOMEM; - ret = clk_bulk_get(dev, num_clks, clks); + if (optional) + ret = clk_bulk_get_optional(dev, num_clks, clks); + else + ret = clk_bulk_get(dev, num_clks, clks); if (!ret) { devres->clks = clks; devres->num_clks = num_clks; @@ -74,8 +77,21 @@ int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, return ret; } + +int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __devm_clk_bulk_get(dev, num_clks, clks, false); +} EXPORT_SYMBOL_GPL(devm_clk_bulk_get); +int __must_check devm_clk_bulk_get_optional(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __devm_clk_bulk_get(dev, num_clks, clks, true); +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get_optional); + int __must_check devm_clk_bulk_get_all(struct device *dev, struct clk_bulk_data **clks) { diff --git a/drivers/clk/clk-lochnagar.c b/drivers/clk/clk-lochnagar.c index a2f31e58ee48..fa8c91758b1d 100644 --- a/drivers/clk/clk-lochnagar.c +++ b/drivers/clk/clk-lochnagar.c @@ -16,7 +16,6 @@ #include <linux/platform_device.h> #include <linux/regmap.h> -#include <linux/mfd/lochnagar.h> #include <linux/mfd/lochnagar1_regs.h> #include <linux/mfd/lochnagar2_regs.h> @@ -40,48 +39,46 @@ struct lochnagar_clk { struct lochnagar_clk_priv { struct device *dev; struct regmap *regmap; - enum lochnagar_type type; - - const char **parents; - unsigned int nparents; struct lochnagar_clk lclks[LOCHNAGAR_NUM_CLOCKS]; }; -static const char * const lochnagar1_clk_parents[] = { - "ln-none", - "ln-spdif-mclk", - "ln-psia1-mclk", - "ln-psia2-mclk", - "ln-cdc-clkout", - "ln-dsp-clkout", - "ln-pmic-32k", - "ln-gf-mclk1", - "ln-gf-mclk3", - "ln-gf-mclk2", - "ln-gf-mclk4", +#define LN_PARENT(NAME) { .name = NAME, .fw_name = NAME } + +static const struct clk_parent_data lochnagar1_clk_parents[] = { + LN_PARENT("ln-none"), + LN_PARENT("ln-spdif-mclk"), + LN_PARENT("ln-psia1-mclk"), + LN_PARENT("ln-psia2-mclk"), + LN_PARENT("ln-cdc-clkout"), + LN_PARENT("ln-dsp-clkout"), + LN_PARENT("ln-pmic-32k"), + LN_PARENT("ln-gf-mclk1"), + LN_PARENT("ln-gf-mclk3"), + LN_PARENT("ln-gf-mclk2"), + LN_PARENT("ln-gf-mclk4"), }; -static const char * const lochnagar2_clk_parents[] = { - "ln-none", - "ln-cdc-clkout", - "ln-dsp-clkout", - "ln-pmic-32k", - "ln-spdif-mclk", - "ln-clk-12m", - "ln-clk-11m", - "ln-clk-24m", - "ln-clk-22m", - "ln-clk-8m", - "ln-usb-clk-24m", - "ln-gf-mclk1", - "ln-gf-mclk3", - "ln-gf-mclk2", - "ln-psia1-mclk", - "ln-psia2-mclk", - "ln-spdif-clkout", - "ln-adat-mclk", - "ln-usb-clk-12m", +static const struct clk_parent_data lochnagar2_clk_parents[] = { + LN_PARENT("ln-none"), + LN_PARENT("ln-cdc-clkout"), + LN_PARENT("ln-dsp-clkout"), + LN_PARENT("ln-pmic-32k"), + LN_PARENT("ln-spdif-mclk"), + LN_PARENT("ln-clk-12m"), + LN_PARENT("ln-clk-11m"), + LN_PARENT("ln-clk-24m"), + LN_PARENT("ln-clk-22m"), + LN_PARENT("ln-clk-8m"), + LN_PARENT("ln-usb-clk-24m"), + LN_PARENT("ln-gf-mclk1"), + LN_PARENT("ln-gf-mclk3"), + LN_PARENT("ln-gf-mclk2"), + LN_PARENT("ln-psia1-mclk"), + LN_PARENT("ln-psia2-mclk"), + LN_PARENT("ln-spdif-clkout"), + LN_PARENT("ln-adat-mclk"), + LN_PARENT("ln-usb-clk-12m"), }; #define LN1_CLK(ID, NAME, REG) \ @@ -122,6 +119,24 @@ static const struct lochnagar_clk lochnagar2_clks[LOCHNAGAR_NUM_CLOCKS] = { LN2_CLK(SOUNDCARD_MCLK, "ln-soundcard-mclk"), }; +struct lochnagar_config { + const struct clk_parent_data *parents; + int nparents; + const struct lochnagar_clk *clks; +}; + +static const struct lochnagar_config lochnagar1_conf = { + .parents = lochnagar1_clk_parents, + .nparents = ARRAY_SIZE(lochnagar1_clk_parents), + .clks = lochnagar1_clks, +}; + +static const struct lochnagar_config lochnagar2_conf = { + .parents = lochnagar2_clk_parents, + .nparents = ARRAY_SIZE(lochnagar2_clk_parents), + .clks = lochnagar2_clks, +}; + static inline struct lochnagar_clk *lochnagar_hw_to_lclk(struct clk_hw *hw) { return container_of(hw, struct lochnagar_clk, hw); @@ -183,7 +198,7 @@ static u8 lochnagar_clk_get_parent(struct clk_hw *hw) if (ret < 0) { dev_dbg(priv->dev, "Failed to read parent of %s: %d\n", lclk->name, ret); - return priv->nparents; + return hw->init->num_parents; } val &= lclk->src_mask; @@ -198,46 +213,6 @@ static const struct clk_ops lochnagar_clk_ops = { .get_parent = lochnagar_clk_get_parent, }; -static int lochnagar_init_parents(struct lochnagar_clk_priv *priv) -{ - struct device_node *np = priv->dev->of_node; - int i, j; - - switch (priv->type) { - case LOCHNAGAR1: - memcpy(priv->lclks, lochnagar1_clks, sizeof(lochnagar1_clks)); - - priv->nparents = ARRAY_SIZE(lochnagar1_clk_parents); - priv->parents = devm_kmemdup(priv->dev, lochnagar1_clk_parents, - sizeof(lochnagar1_clk_parents), - GFP_KERNEL); - break; - case LOCHNAGAR2: - memcpy(priv->lclks, lochnagar2_clks, sizeof(lochnagar2_clks)); - - priv->nparents = ARRAY_SIZE(lochnagar2_clk_parents); - priv->parents = devm_kmemdup(priv->dev, lochnagar2_clk_parents, - sizeof(lochnagar2_clk_parents), - GFP_KERNEL); - break; - default: - dev_err(priv->dev, "Unknown Lochnagar type: %d\n", priv->type); - return -EINVAL; - } - - if (!priv->parents) - return -ENOMEM; - - for (i = 0; i < priv->nparents; i++) { - j = of_property_match_string(np, "clock-names", - priv->parents[i]); - if (j >= 0) - priv->parents[i] = of_clk_get_parent_name(np, j); - } - - return 0; -} - static struct clk_hw * lochnagar_of_clk_hw_get(struct of_phandle_args *clkspec, void *data) { @@ -252,16 +227,42 @@ lochnagar_of_clk_hw_get(struct of_phandle_args *clkspec, void *data) return &priv->lclks[idx].hw; } -static int lochnagar_init_clks(struct lochnagar_clk_priv *priv) +static const struct of_device_id lochnagar_of_match[] = { + { .compatible = "cirrus,lochnagar1-clk", .data = &lochnagar1_conf }, + { .compatible = "cirrus,lochnagar2-clk", .data = &lochnagar2_conf }, + {} +}; +MODULE_DEVICE_TABLE(of, lochnagar_of_match); + +static int lochnagar_clk_probe(struct platform_device *pdev) { struct clk_init_data clk_init = { .ops = &lochnagar_clk_ops, - .parent_names = priv->parents, - .num_parents = priv->nparents, }; + struct device *dev = &pdev->dev; + struct lochnagar_clk_priv *priv; + const struct of_device_id *of_id; struct lochnagar_clk *lclk; + struct lochnagar_config *conf; int ret, i; + of_id = of_match_device(lochnagar_of_match, dev); + if (!of_id) + return -EINVAL; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + priv->regmap = dev_get_regmap(dev->parent, NULL); + conf = (struct lochnagar_config *)of_id->data; + + memcpy(priv->lclks, conf->clks, sizeof(priv->lclks)); + + clk_init.parent_data = conf->parents; + clk_init.num_parents = conf->nparents; + for (i = 0; i < ARRAY_SIZE(priv->lclks); i++) { lclk = &priv->lclks[i]; @@ -273,55 +274,21 @@ static int lochnagar_init_clks(struct lochnagar_clk_priv *priv) lclk->priv = priv; lclk->hw.init = &clk_init; - ret = devm_clk_hw_register(priv->dev, &lclk->hw); + ret = devm_clk_hw_register(dev, &lclk->hw); if (ret) { - dev_err(priv->dev, "Failed to register %s: %d\n", + dev_err(dev, "Failed to register %s: %d\n", lclk->name, ret); return ret; } } - ret = devm_of_clk_add_hw_provider(priv->dev, lochnagar_of_clk_hw_get, - priv); + ret = devm_of_clk_add_hw_provider(dev, lochnagar_of_clk_hw_get, priv); if (ret < 0) - dev_err(priv->dev, "Failed to register provider: %d\n", ret); + dev_err(dev, "Failed to register provider: %d\n", ret); return ret; } -static const struct of_device_id lochnagar_of_match[] = { - { .compatible = "cirrus,lochnagar1-clk", .data = (void *)LOCHNAGAR1 }, - { .compatible = "cirrus,lochnagar2-clk", .data = (void *)LOCHNAGAR2 }, - {} -}; -MODULE_DEVICE_TABLE(of, lochnagar_of_match); - -static int lochnagar_clk_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct lochnagar_clk_priv *priv; - const struct of_device_id *of_id; - int ret; - - of_id = of_match_device(lochnagar_of_match, dev); - if (!of_id) - return -EINVAL; - - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->dev = dev; - priv->regmap = dev_get_regmap(dev->parent, NULL); - priv->type = (enum lochnagar_type)of_id->data; - - ret = lochnagar_init_parents(priv); - if (ret) - return ret; - - return lochnagar_init_clks(priv); -} - static struct platform_driver lochnagar_clk_driver = { .driver = { .name = "lochnagar-clk", diff --git a/drivers/clk/clk-pwm.c b/drivers/clk/clk-pwm.c index 5f0490b8f6cb..87fe0b0e01a3 100644 --- a/drivers/clk/clk-pwm.c +++ b/drivers/clk/clk-pwm.c @@ -44,10 +44,24 @@ static unsigned long clk_pwm_recalc_rate(struct clk_hw *hw, return clk_pwm->fixed_rate; } +static int clk_pwm_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) +{ + struct clk_pwm *clk_pwm = to_clk_pwm(hw); + struct pwm_state state; + + pwm_get_state(clk_pwm->pwm, &state); + + duty->num = state.duty_cycle; + duty->den = state.period; + + return 0; +} + static const struct clk_ops clk_pwm_ops = { .prepare = clk_pwm_prepare, .unprepare = clk_pwm_unprepare, .recalc_rate = clk_pwm_recalc_rate, + .get_duty_cycle = clk_pwm_get_duty_cycle, }; static int clk_pwm_probe(struct platform_device *pdev) diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index dd93d3acc67d..07f3b252f3e0 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -635,6 +635,17 @@ static const struct clockgen_chipinfo chipinfo[] = { .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { + .compat = "fsl,lx2160a-clockgen", + .cmux_groups = { + &clockgen2_cmux_cga12, &clockgen2_cmux_cgb + }, + .cmux_to_group = { + 0, 0, 0, 0, 1, 1, 1, 1, -1 + }, + .pll_mask = 0x37, + .flags = CG_VER3 | CG_LITTLE_ENDIAN, + }, + { .compat = "fsl,p2041-clockgen", .guts_compat = "fsl,qoriq-device-config-1.0", .init_periph = p2041_init_periph, @@ -1493,6 +1504,7 @@ CLK_OF_DECLARE(qoriq_clockgen_ls1043a, "fsl,ls1043a-clockgen", clockgen_init); CLK_OF_DECLARE(qoriq_clockgen_ls1046a, "fsl,ls1046a-clockgen", clockgen_init); CLK_OF_DECLARE(qoriq_clockgen_ls1088a, "fsl,ls1088a-clockgen", clockgen_init); CLK_OF_DECLARE(qoriq_clockgen_ls2080a, "fsl,ls2080a-clockgen", clockgen_init); +CLK_OF_DECLARE(qoriq_clockgen_lx2160a, "fsl,lx2160a-clockgen", clockgen_init); CLK_OF_DECLARE(qoriq_clockgen_p2041, "fsl,p2041-clockgen", clockgen_init); CLK_OF_DECLARE(qoriq_clockgen_p3041, "fsl,p3041-clockgen", clockgen_init); CLK_OF_DECLARE(qoriq_clockgen_p4080, "fsl,p4080-clockgen", clockgen_init); diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c new file mode 100644 index 000000000000..72424eb7e5f8 --- /dev/null +++ b/drivers/clk/clk-si5341.c @@ -0,0 +1,1346 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for Silicon Labs Si5341/Si5340 Clock generator + * Copyright (C) 2019 Topic Embedded Products + * Author: Mike Looijmans <mike.looijmans@topic.nl> + */ + +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/delay.h> +#include <linux/gcd.h> +#include <linux/math64.h> +#include <linux/i2c.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <asm/unaligned.h> + +#define SI5341_MAX_NUM_OUTPUTS 10 +#define SI5340_MAX_NUM_OUTPUTS 4 + +#define SI5341_NUM_SYNTH 5 +#define SI5340_NUM_SYNTH 4 + +/* Range of the synthesizer fractional divider */ +#define SI5341_SYNTH_N_MIN 10 +#define SI5341_SYNTH_N_MAX 4095 + +/* The chip can get its input clock from 3 input pins or an XTAL */ + +/* There is one PLL running at 13500–14256 MHz */ +#define SI5341_PLL_VCO_MIN 13500000000ull +#define SI5341_PLL_VCO_MAX 14256000000ull + +/* The 5 frequency synthesizers obtain their input from the PLL */ +struct clk_si5341_synth { + struct clk_hw hw; + struct clk_si5341 *data; + u8 index; +}; +#define to_clk_si5341_synth(_hw) \ + container_of(_hw, struct clk_si5341_synth, hw) + +/* The output stages can be connected to any synth (full mux) */ +struct clk_si5341_output { + struct clk_hw hw; + struct clk_si5341 *data; + u8 index; +}; +#define to_clk_si5341_output(_hw) \ + container_of(_hw, struct clk_si5341_output, hw) + +struct clk_si5341 { + struct clk_hw hw; + struct regmap *regmap; + struct i2c_client *i2c_client; + struct clk_si5341_synth synth[SI5341_NUM_SYNTH]; + struct clk_si5341_output clk[SI5341_MAX_NUM_OUTPUTS]; + struct clk *pxtal; + const char *pxtal_name; + const u16 *reg_output_offset; + const u16 *reg_rdiv_offset; + u64 freq_vco; /* 13500–14256 MHz */ + u8 num_outputs; + u8 num_synth; +}; +#define to_clk_si5341(_hw) container_of(_hw, struct clk_si5341, hw) + +struct clk_si5341_output_config { + u8 out_format_drv_bits; + u8 out_cm_ampl_bits; + bool synth_master; + bool always_on; +}; + +#define SI5341_PAGE 0x0001 +#define SI5341_PN_BASE 0x0002 +#define SI5341_DEVICE_REV 0x0005 +#define SI5341_STATUS 0x000C +#define SI5341_SOFT_RST 0x001C + +/* Input dividers (48-bit) */ +#define SI5341_IN_PDIV(x) (0x0208 + ((x) * 10)) +#define SI5341_IN_PSET(x) (0x020E + ((x) * 10)) + +/* PLL configuration */ +#define SI5341_PLL_M_NUM 0x0235 +#define SI5341_PLL_M_DEN 0x023B + +/* Output configuration */ +#define SI5341_OUT_CONFIG(output) \ + ((output)->data->reg_output_offset[(output)->index]) +#define SI5341_OUT_FORMAT(output) (SI5341_OUT_CONFIG(output) + 1) +#define SI5341_OUT_CM(output) (SI5341_OUT_CONFIG(output) + 2) +#define SI5341_OUT_MUX_SEL(output) (SI5341_OUT_CONFIG(output) + 3) +#define SI5341_OUT_R_REG(output) \ + ((output)->data->reg_rdiv_offset[(output)->index]) + +/* Synthesize N divider */ +#define SI5341_SYNTH_N_NUM(x) (0x0302 + ((x) * 11)) +#define SI5341_SYNTH_N_DEN(x) (0x0308 + ((x) * 11)) +#define SI5341_SYNTH_N_UPD(x) (0x030C + ((x) * 11)) + +/* Synthesizer output enable, phase bypass, power mode */ +#define SI5341_SYNTH_N_CLK_TO_OUTX_EN 0x0A03 +#define SI5341_SYNTH_N_PIBYP 0x0A04 +#define SI5341_SYNTH_N_PDNB 0x0A05 +#define SI5341_SYNTH_N_CLK_DIS 0x0B4A + +#define SI5341_REGISTER_MAX 0xBFF + +/* SI5341_OUT_CONFIG bits */ +#define SI5341_OUT_CFG_PDN BIT(0) +#define SI5341_OUT_CFG_OE BIT(1) +#define SI5341_OUT_CFG_RDIV_FORCE2 BIT(2) + +/* Static configuration (to be moved to firmware) */ +struct si5341_reg_default { + u16 address; + u8 value; +}; + +/* Output configuration registers 0..9 are not quite logically organized */ +static const u16 si5341_reg_output_offset[] = { + 0x0108, + 0x010D, + 0x0112, + 0x0117, + 0x011C, + 0x0121, + 0x0126, + 0x012B, + 0x0130, + 0x013A, +}; + +static const u16 si5340_reg_output_offset[] = { + 0x0112, + 0x0117, + 0x0126, + 0x012B, +}; + +/* The location of the R divider registers */ +static const u16 si5341_reg_rdiv_offset[] = { + 0x024A, + 0x024D, + 0x0250, + 0x0253, + 0x0256, + 0x0259, + 0x025C, + 0x025F, + 0x0262, + 0x0268, +}; +static const u16 si5340_reg_rdiv_offset[] = { + 0x0250, + 0x0253, + 0x025C, + 0x025F, +}; + +/* + * Programming sequence from ClockBuilder, settings to initialize the system + * using only the XTAL input, without pre-divider. + * This also contains settings that aren't mentioned anywhere in the datasheet. + * The "known" settings like synth and output configuration are done later. + */ +static const struct si5341_reg_default si5341_reg_defaults[] = { + { 0x0017, 0x3A }, /* INT mask (disable interrupts) */ + { 0x0018, 0xFF }, /* INT mask */ + { 0x0021, 0x0F }, /* Select XTAL as input */ + { 0x0022, 0x00 }, /* Not in datasheet */ + { 0x002B, 0x02 }, /* SPI config */ + { 0x002C, 0x20 }, /* LOS enable for XTAL */ + { 0x002D, 0x00 }, /* LOS timing */ + { 0x002E, 0x00 }, + { 0x002F, 0x00 }, + { 0x0030, 0x00 }, + { 0x0031, 0x00 }, + { 0x0032, 0x00 }, + { 0x0033, 0x00 }, + { 0x0034, 0x00 }, + { 0x0035, 0x00 }, + { 0x0036, 0x00 }, + { 0x0037, 0x00 }, + { 0x0038, 0x00 }, /* LOS setting (thresholds) */ + { 0x0039, 0x00 }, + { 0x003A, 0x00 }, + { 0x003B, 0x00 }, + { 0x003C, 0x00 }, + { 0x003D, 0x00 }, /* LOS setting (thresholds) end */ + { 0x0041, 0x00 }, /* LOS0_DIV_SEL */ + { 0x0042, 0x00 }, /* LOS1_DIV_SEL */ + { 0x0043, 0x00 }, /* LOS2_DIV_SEL */ + { 0x0044, 0x00 }, /* LOS3_DIV_SEL */ + { 0x009E, 0x00 }, /* Not in datasheet */ + { 0x0102, 0x01 }, /* Enable outputs */ + { 0x013F, 0x00 }, /* Not in datasheet */ + { 0x0140, 0x00 }, /* Not in datasheet */ + { 0x0141, 0x40 }, /* OUT LOS */ + { 0x0202, 0x00 }, /* XAXB_FREQ_OFFSET (=0)*/ + { 0x0203, 0x00 }, + { 0x0204, 0x00 }, + { 0x0205, 0x00 }, + { 0x0206, 0x00 }, /* PXAXB (2^x) */ + { 0x0208, 0x00 }, /* Px divider setting (usually 0) */ + { 0x0209, 0x00 }, + { 0x020A, 0x00 }, + { 0x020B, 0x00 }, + { 0x020C, 0x00 }, + { 0x020D, 0x00 }, + { 0x020E, 0x00 }, + { 0x020F, 0x00 }, + { 0x0210, 0x00 }, + { 0x0211, 0x00 }, + { 0x0212, 0x00 }, + { 0x0213, 0x00 }, + { 0x0214, 0x00 }, + { 0x0215, 0x00 }, + { 0x0216, 0x00 }, + { 0x0217, 0x00 }, + { 0x0218, 0x00 }, + { 0x0219, 0x00 }, + { 0x021A, 0x00 }, + { 0x021B, 0x00 }, + { 0x021C, 0x00 }, + { 0x021D, 0x00 }, + { 0x021E, 0x00 }, + { 0x021F, 0x00 }, + { 0x0220, 0x00 }, + { 0x0221, 0x00 }, + { 0x0222, 0x00 }, + { 0x0223, 0x00 }, + { 0x0224, 0x00 }, + { 0x0225, 0x00 }, + { 0x0226, 0x00 }, + { 0x0227, 0x00 }, + { 0x0228, 0x00 }, + { 0x0229, 0x00 }, + { 0x022A, 0x00 }, + { 0x022B, 0x00 }, + { 0x022C, 0x00 }, + { 0x022D, 0x00 }, + { 0x022E, 0x00 }, + { 0x022F, 0x00 }, /* Px divider setting (usually 0) end */ + { 0x026B, 0x00 }, /* DESIGN_ID (ASCII string) */ + { 0x026C, 0x00 }, + { 0x026D, 0x00 }, + { 0x026E, 0x00 }, + { 0x026F, 0x00 }, + { 0x0270, 0x00 }, + { 0x0271, 0x00 }, + { 0x0272, 0x00 }, /* DESIGN_ID (ASCII string) end */ + { 0x0339, 0x1F }, /* N_FSTEP_MSK */ + { 0x033B, 0x00 }, /* Nx_FSTEPW (Frequency step) */ + { 0x033C, 0x00 }, + { 0x033D, 0x00 }, + { 0x033E, 0x00 }, + { 0x033F, 0x00 }, + { 0x0340, 0x00 }, + { 0x0341, 0x00 }, + { 0x0342, 0x00 }, + { 0x0343, 0x00 }, + { 0x0344, 0x00 }, + { 0x0345, 0x00 }, + { 0x0346, 0x00 }, + { 0x0347, 0x00 }, + { 0x0348, 0x00 }, + { 0x0349, 0x00 }, + { 0x034A, 0x00 }, + { 0x034B, 0x00 }, + { 0x034C, 0x00 }, + { 0x034D, 0x00 }, + { 0x034E, 0x00 }, + { 0x034F, 0x00 }, + { 0x0350, 0x00 }, + { 0x0351, 0x00 }, + { 0x0352, 0x00 }, + { 0x0353, 0x00 }, + { 0x0354, 0x00 }, + { 0x0355, 0x00 }, + { 0x0356, 0x00 }, + { 0x0357, 0x00 }, + { 0x0358, 0x00 }, /* Nx_FSTEPW (Frequency step) end */ + { 0x0359, 0x00 }, /* Nx_DELAY */ + { 0x035A, 0x00 }, + { 0x035B, 0x00 }, + { 0x035C, 0x00 }, + { 0x035D, 0x00 }, + { 0x035E, 0x00 }, + { 0x035F, 0x00 }, + { 0x0360, 0x00 }, + { 0x0361, 0x00 }, + { 0x0362, 0x00 }, /* Nx_DELAY end */ + { 0x0802, 0x00 }, /* Not in datasheet */ + { 0x0803, 0x00 }, /* Not in datasheet */ + { 0x0804, 0x00 }, /* Not in datasheet */ + { 0x090E, 0x02 }, /* XAXB_EXTCLK_EN=0 XAXB_PDNB=1 (use XTAL) */ + { 0x091C, 0x04 }, /* ZDM_EN=4 (Normal mode) */ + { 0x0943, 0x00 }, /* IO_VDD_SEL=0 (0=1v8, use 1=3v3) */ + { 0x0949, 0x00 }, /* IN_EN (disable input clocks) */ + { 0x094A, 0x00 }, /* INx_TO_PFD_EN (disabled) */ + { 0x0A02, 0x00 }, /* Not in datasheet */ + { 0x0B44, 0x0F }, /* PDIV_ENB (datasheet does not mention what it is) */ +}; + +/* Read and interpret a 44-bit followed by a 32-bit value in the regmap */ +static int si5341_decode_44_32(struct regmap *regmap, unsigned int reg, + u64 *val1, u32 *val2) +{ + int err; + u8 r[10]; + + err = regmap_bulk_read(regmap, reg, r, 10); + if (err < 0) + return err; + + *val1 = ((u64)((r[5] & 0x0f) << 8 | r[4]) << 32) | + (get_unaligned_le32(r)); + *val2 = get_unaligned_le32(&r[6]); + + return 0; +} + +static int si5341_encode_44_32(struct regmap *regmap, unsigned int reg, + u64 n_num, u32 n_den) +{ + u8 r[10]; + + /* Shift left as far as possible without overflowing */ + while (!(n_num & BIT_ULL(43)) && !(n_den & BIT(31))) { + n_num <<= 1; + n_den <<= 1; + } + + /* 44 bits (6 bytes) numerator */ + put_unaligned_le32(n_num, r); + r[4] = (n_num >> 32) & 0xff; + r[5] = (n_num >> 40) & 0x0f; + /* 32 bits denominator */ + put_unaligned_le32(n_den, &r[6]); + + /* Program the fraction */ + return regmap_bulk_write(regmap, reg, r, sizeof(r)); +} + +/* VCO, we assume it runs at a constant frequency */ +static unsigned long si5341_clk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_si5341 *data = to_clk_si5341(hw); + int err; + u64 res; + u64 m_num; + u32 m_den; + unsigned int shift; + + /* Assume that PDIV is not being used, just read the PLL setting */ + err = si5341_decode_44_32(data->regmap, SI5341_PLL_M_NUM, + &m_num, &m_den); + if (err < 0) + return 0; + + if (!m_num || !m_den) + return 0; + + /* + * Though m_num is 64-bit, only the upper bits are actually used. While + * calculating m_num and m_den, they are shifted as far as possible to + * the left. To avoid 96-bit division here, we just shift them back so + * we can do with just 64 bits. + */ + shift = 0; + res = m_num; + while (res & 0xffff00000000ULL) { + ++shift; + res >>= 1; + } + res *= parent_rate; + do_div(res, (m_den >> shift)); + + /* We cannot return the actual frequency in 32 bit, store it locally */ + data->freq_vco = res; + + /* Report kHz since the value is out of range */ + do_div(res, 1000); + + return (unsigned long)res; +} + +static const struct clk_ops si5341_clk_ops = { + .recalc_rate = si5341_clk_recalc_rate, +}; + +/* Synthesizers, there are 5 synthesizers that connect to any of the outputs */ + +/* The synthesizer is on if all power and enable bits are set */ +static int si5341_synth_clk_is_on(struct clk_hw *hw) +{ + struct clk_si5341_synth *synth = to_clk_si5341_synth(hw); + int err; + u32 val; + u8 index = synth->index; + + err = regmap_read(synth->data->regmap, + SI5341_SYNTH_N_CLK_TO_OUTX_EN, &val); + if (err < 0) + return 0; + + if (!(val & BIT(index))) + return 0; + + err = regmap_read(synth->data->regmap, SI5341_SYNTH_N_PDNB, &val); + if (err < 0) + return 0; + + if (!(val & BIT(index))) + return 0; + + /* This bit must be 0 for the synthesizer to receive clock input */ + err = regmap_read(synth->data->regmap, SI5341_SYNTH_N_CLK_DIS, &val); + if (err < 0) + return 0; + + return !(val & BIT(index)); +} + +static void si5341_synth_clk_unprepare(struct clk_hw *hw) +{ + struct clk_si5341_synth *synth = to_clk_si5341_synth(hw); + u8 index = synth->index; /* In range 0..5 */ + u8 mask = BIT(index); + + /* Disable output */ + regmap_update_bits(synth->data->regmap, + SI5341_SYNTH_N_CLK_TO_OUTX_EN, mask, 0); + /* Power down */ + regmap_update_bits(synth->data->regmap, + SI5341_SYNTH_N_PDNB, mask, 0); + /* Disable clock input to synth (set to 1 to disable) */ + regmap_update_bits(synth->data->regmap, + SI5341_SYNTH_N_CLK_DIS, mask, mask); +} + +static int si5341_synth_clk_prepare(struct clk_hw *hw) +{ + struct clk_si5341_synth *synth = to_clk_si5341_synth(hw); + int err; + u8 index = synth->index; + u8 mask = BIT(index); + + /* Power up */ + err = regmap_update_bits(synth->data->regmap, + SI5341_SYNTH_N_PDNB, mask, mask); + if (err < 0) + return err; + + /* Enable clock input to synth (set bit to 0 to enable) */ + err = regmap_update_bits(synth->data->regmap, + SI5341_SYNTH_N_CLK_DIS, mask, 0); + if (err < 0) + return err; + + /* Enable output */ + return regmap_update_bits(synth->data->regmap, + SI5341_SYNTH_N_CLK_TO_OUTX_EN, mask, mask); +} + +/* Synth clock frequency: Fvco * n_den / n_den, with Fvco in 13500-14256 MHz */ +static unsigned long si5341_synth_clk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_si5341_synth *synth = to_clk_si5341_synth(hw); + u64 f; + u64 n_num; + u32 n_den; + int err; + + err = si5341_decode_44_32(synth->data->regmap, + SI5341_SYNTH_N_NUM(synth->index), &n_num, &n_den); + if (err < 0) + return err; + + /* + * n_num and n_den are shifted left as much as possible, so to prevent + * overflow in 64-bit math, we shift n_den 4 bits to the right + */ + f = synth->data->freq_vco; + f *= n_den >> 4; + + /* Now we need to to 64-bit division: f/n_num */ + /* And compensate for the 4 bits we dropped */ + f = div64_u64(f, (n_num >> 4)); + + return f; +} + +static long si5341_synth_clk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct clk_si5341_synth *synth = to_clk_si5341_synth(hw); + u64 f; + + /* The synthesizer accuracy is such that anything in range will work */ + f = synth->data->freq_vco; + do_div(f, SI5341_SYNTH_N_MAX); + if (rate < f) + return f; + + f = synth->data->freq_vco; + do_div(f, SI5341_SYNTH_N_MIN); + if (rate > f) + return f; + + return rate; +} + +static int si5341_synth_program(struct clk_si5341_synth *synth, + u64 n_num, u32 n_den, bool is_integer) +{ + int err; + u8 index = synth->index; + + err = si5341_encode_44_32(synth->data->regmap, + SI5341_SYNTH_N_NUM(index), n_num, n_den); + + err = regmap_update_bits(synth->data->regmap, + SI5341_SYNTH_N_PIBYP, BIT(index), is_integer ? BIT(index) : 0); + if (err < 0) + return err; + + return regmap_write(synth->data->regmap, + SI5341_SYNTH_N_UPD(index), 0x01); +} + + +static int si5341_synth_clk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_si5341_synth *synth = to_clk_si5341_synth(hw); + u64 n_num; + u32 n_den; + u32 r; + u32 g; + bool is_integer; + + n_num = synth->data->freq_vco; + n_den = rate; + + /* see if there's an integer solution */ + r = do_div(n_num, rate); + is_integer = (r == 0); + if (is_integer) { + /* Integer divider equal to n_num */ + n_den = 1; + } else { + /* Calculate a fractional solution */ + g = gcd(r, rate); + n_den = rate / g; + n_num *= n_den; + n_num += r / g; + } + + dev_dbg(&synth->data->i2c_client->dev, + "%s(%u): n=0x%llx d=0x%x %s\n", __func__, + synth->index, n_num, n_den, + is_integer ? "int" : "frac"); + + return si5341_synth_program(synth, n_num, n_den, is_integer); +} + +static const struct clk_ops si5341_synth_clk_ops = { + .is_prepared = si5341_synth_clk_is_on, + .prepare = si5341_synth_clk_prepare, + .unprepare = si5341_synth_clk_unprepare, + .recalc_rate = si5341_synth_clk_recalc_rate, + .round_rate = si5341_synth_clk_round_rate, + .set_rate = si5341_synth_clk_set_rate, +}; + +static int si5341_output_clk_is_on(struct clk_hw *hw) +{ + struct clk_si5341_output *output = to_clk_si5341_output(hw); + int err; + u32 val; + + err = regmap_read(output->data->regmap, + SI5341_OUT_CONFIG(output), &val); + if (err < 0) + return err; + + /* Bit 0=PDN, 1=OE so only a value of 0x2 enables the output */ + return (val & 0x03) == SI5341_OUT_CFG_OE; +} + +/* Disables and then powers down the output */ +static void si5341_output_clk_unprepare(struct clk_hw *hw) +{ + struct clk_si5341_output *output = to_clk_si5341_output(hw); + + regmap_update_bits(output->data->regmap, + SI5341_OUT_CONFIG(output), + SI5341_OUT_CFG_OE, 0); + regmap_update_bits(output->data->regmap, + SI5341_OUT_CONFIG(output), + SI5341_OUT_CFG_PDN, SI5341_OUT_CFG_PDN); +} + +/* Powers up and then enables the output */ +static int si5341_output_clk_prepare(struct clk_hw *hw) +{ + struct clk_si5341_output *output = to_clk_si5341_output(hw); + int err; + + err = regmap_update_bits(output->data->regmap, + SI5341_OUT_CONFIG(output), + SI5341_OUT_CFG_PDN, 0); + if (err < 0) + return err; + + return regmap_update_bits(output->data->regmap, + SI5341_OUT_CONFIG(output), + SI5341_OUT_CFG_OE, SI5341_OUT_CFG_OE); +} + +static unsigned long si5341_output_clk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_si5341_output *output = to_clk_si5341_output(hw); + int err; + u32 val; + u32 r_divider; + u8 r[3]; + + err = regmap_bulk_read(output->data->regmap, + SI5341_OUT_R_REG(output), r, 3); + if (err < 0) + return err; + + /* Calculate value as 24-bit integer*/ + r_divider = r[2] << 16 | r[1] << 8 | r[0]; + + /* If Rx_REG is zero, the divider is disabled, so return a "0" rate */ + if (!r_divider) + return 0; + + /* Divider is 2*(Rx_REG+1) */ + r_divider += 1; + r_divider <<= 1; + + err = regmap_read(output->data->regmap, + SI5341_OUT_CONFIG(output), &val); + if (err < 0) + return err; + + if (val & SI5341_OUT_CFG_RDIV_FORCE2) + r_divider = 2; + + return parent_rate / r_divider; +} + +static long si5341_output_clk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + unsigned long r; + + r = *parent_rate >> 1; + + /* If rate is an even divisor, no changes to parent required */ + if (r && !(r % rate)) + return (long)rate; + + if (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) { + if (rate > 200000000) { + /* minimum r-divider is 2 */ + r = 2; + } else { + /* Take a parent frequency near 400 MHz */ + r = (400000000u / rate) & ~1; + } + *parent_rate = r * rate; + } else { + /* We cannot change our parent's rate, report what we can do */ + r /= rate; + rate = *parent_rate / (r << 1); + } + + return rate; +} + +static int si5341_output_clk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_si5341_output *output = to_clk_si5341_output(hw); + /* Frequency divider is (r_div + 1) * 2 */ + u32 r_div = (parent_rate / rate) >> 1; + int err; + u8 r[3]; + + if (r_div <= 1) + r_div = 0; + else if (r_div >= BIT(24)) + r_div = BIT(24) - 1; + else + --r_div; + + /* For a value of "2", we set the "OUT0_RDIV_FORCE2" bit */ + err = regmap_update_bits(output->data->regmap, + SI5341_OUT_CONFIG(output), + SI5341_OUT_CFG_RDIV_FORCE2, + (r_div == 0) ? SI5341_OUT_CFG_RDIV_FORCE2 : 0); + if (err < 0) + return err; + + /* Always write Rx_REG, because a zero value disables the divider */ + r[0] = r_div ? (r_div & 0xff) : 1; + r[1] = (r_div >> 8) & 0xff; + r[2] = (r_div >> 16) & 0xff; + err = regmap_bulk_write(output->data->regmap, + SI5341_OUT_R_REG(output), r, 3); + + return 0; +} + +static int si5341_output_reparent(struct clk_si5341_output *output, u8 index) +{ + return regmap_update_bits(output->data->regmap, + SI5341_OUT_MUX_SEL(output), 0x07, index); +} + +static int si5341_output_set_parent(struct clk_hw *hw, u8 index) +{ + struct clk_si5341_output *output = to_clk_si5341_output(hw); + + if (index >= output->data->num_synth) + return -EINVAL; + + return si5341_output_reparent(output, index); +} + +static u8 si5341_output_get_parent(struct clk_hw *hw) +{ + struct clk_si5341_output *output = to_clk_si5341_output(hw); + int err; + u32 val; + + err = regmap_read(output->data->regmap, + SI5341_OUT_MUX_SEL(output), &val); + + return val & 0x7; +} + +static const struct clk_ops si5341_output_clk_ops = { + .is_prepared = si5341_output_clk_is_on, + .prepare = si5341_output_clk_prepare, + .unprepare = si5341_output_clk_unprepare, + .recalc_rate = si5341_output_clk_recalc_rate, + .round_rate = si5341_output_clk_round_rate, + .set_rate = si5341_output_clk_set_rate, + .set_parent = si5341_output_set_parent, + .get_parent = si5341_output_get_parent, +}; + +/* + * The chip can be bought in a pre-programmed version, or one can program the + * NVM in the chip to boot up in a preset mode. This routine tries to determine + * if that's the case, or if we need to reset and program everything from + * scratch. Returns negative error, or true/false. + */ +static int si5341_is_programmed_already(struct clk_si5341 *data) +{ + int err; + u8 r[4]; + + /* Read the PLL divider value, it must have a non-zero value */ + err = regmap_bulk_read(data->regmap, SI5341_PLL_M_DEN, + r, ARRAY_SIZE(r)); + if (err < 0) + return err; + + return !!get_unaligned_le32(r); +} + +static struct clk_hw * +of_clk_si5341_get(struct of_phandle_args *clkspec, void *_data) +{ + struct clk_si5341 *data = _data; + unsigned int idx = clkspec->args[1]; + unsigned int group = clkspec->args[0]; + + switch (group) { + case 0: + if (idx >= data->num_outputs) { + dev_err(&data->i2c_client->dev, + "invalid output index %u\n", idx); + return ERR_PTR(-EINVAL); + } + return &data->clk[idx].hw; + case 1: + if (idx >= data->num_synth) { + dev_err(&data->i2c_client->dev, + "invalid synthesizer index %u\n", idx); + return ERR_PTR(-EINVAL); + } + return &data->synth[idx].hw; + case 2: + if (idx > 0) { + dev_err(&data->i2c_client->dev, + "invalid PLL index %u\n", idx); + return ERR_PTR(-EINVAL); + } + return &data->hw; + default: + dev_err(&data->i2c_client->dev, "invalid group %u\n", group); + return ERR_PTR(-EINVAL); + } +} + +static int si5341_probe_chip_id(struct clk_si5341 *data) +{ + int err; + u8 reg[4]; + u16 model; + + err = regmap_bulk_read(data->regmap, SI5341_PN_BASE, reg, + ARRAY_SIZE(reg)); + if (err < 0) { + dev_err(&data->i2c_client->dev, "Failed to read chip ID\n"); + return err; + } + + model = get_unaligned_le16(reg); + + dev_info(&data->i2c_client->dev, "Chip: %x Grade: %u Rev: %u\n", + model, reg[2], reg[3]); + + switch (model) { + case 0x5340: + data->num_outputs = SI5340_MAX_NUM_OUTPUTS; + data->num_synth = SI5340_NUM_SYNTH; + data->reg_output_offset = si5340_reg_output_offset; + data->reg_rdiv_offset = si5340_reg_rdiv_offset; + break; + case 0x5341: + data->num_outputs = SI5341_MAX_NUM_OUTPUTS; + data->num_synth = SI5341_NUM_SYNTH; + data->reg_output_offset = si5341_reg_output_offset; + data->reg_rdiv_offset = si5341_reg_rdiv_offset; + break; + default: + dev_err(&data->i2c_client->dev, "Model '%x' not supported\n", + model); + return -EINVAL; + } + + return 0; +} + +/* Read active settings into the regmap cache for later reference */ +static int si5341_read_settings(struct clk_si5341 *data) +{ + int err; + u8 i; + u8 r[10]; + + err = regmap_bulk_read(data->regmap, SI5341_PLL_M_NUM, r, 10); + if (err < 0) + return err; + + err = regmap_bulk_read(data->regmap, + SI5341_SYNTH_N_CLK_TO_OUTX_EN, r, 3); + if (err < 0) + return err; + + err = regmap_bulk_read(data->regmap, + SI5341_SYNTH_N_CLK_DIS, r, 1); + if (err < 0) + return err; + + for (i = 0; i < data->num_synth; ++i) { + err = regmap_bulk_read(data->regmap, + SI5341_SYNTH_N_NUM(i), r, 10); + if (err < 0) + return err; + } + + for (i = 0; i < data->num_outputs; ++i) { + err = regmap_bulk_read(data->regmap, + data->reg_output_offset[i], r, 4); + if (err < 0) + return err; + + err = regmap_bulk_read(data->regmap, + data->reg_rdiv_offset[i], r, 3); + if (err < 0) + return err; + } + + return 0; +} + +static int si5341_write_multiple(struct clk_si5341 *data, + const struct si5341_reg_default *values, unsigned int num_values) +{ + unsigned int i; + int res; + + for (i = 0; i < num_values; ++i) { + res = regmap_write(data->regmap, + values[i].address, values[i].value); + if (res < 0) { + dev_err(&data->i2c_client->dev, + "Failed to write %#x:%#x\n", + values[i].address, values[i].value); + return res; + } + } + + return 0; +} + +static const struct si5341_reg_default si5341_preamble[] = { + { 0x0B25, 0x00 }, + { 0x0502, 0x01 }, + { 0x0505, 0x03 }, + { 0x0957, 0x1F }, + { 0x0B4E, 0x1A }, +}; + +static int si5341_send_preamble(struct clk_si5341 *data) +{ + int res; + u32 revision; + + /* For revision 2 and up, the values are slightly different */ + res = regmap_read(data->regmap, SI5341_DEVICE_REV, &revision); + if (res < 0) + return res; + + /* Write "preamble" as specified by datasheet */ + res = regmap_write(data->regmap, 0xB24, revision < 2 ? 0xD8 : 0xC0); + if (res < 0) + return res; + res = si5341_write_multiple(data, + si5341_preamble, ARRAY_SIZE(si5341_preamble)); + if (res < 0) + return res; + + /* Datasheet specifies a 300ms wait after sending the preamble */ + msleep(300); + + return 0; +} + +/* Perform a soft reset and write post-amble */ +static int si5341_finalize_defaults(struct clk_si5341 *data) +{ + int res; + u32 revision; + + res = regmap_read(data->regmap, SI5341_DEVICE_REV, &revision); + if (res < 0) + return res; + + dev_dbg(&data->i2c_client->dev, "%s rev=%u\n", __func__, revision); + + res = regmap_write(data->regmap, SI5341_SOFT_RST, 0x01); + if (res < 0) + return res; + + /* Datasheet does not explain these nameless registers */ + res = regmap_write(data->regmap, 0xB24, revision < 2 ? 0xDB : 0xC3); + if (res < 0) + return res; + res = regmap_write(data->regmap, 0x0B25, 0x02); + if (res < 0) + return res; + + return 0; +} + + +static const struct regmap_range si5341_regmap_volatile_range[] = { + regmap_reg_range(0x000C, 0x0012), /* Status */ + regmap_reg_range(0x001C, 0x001E), /* reset, finc/fdec */ + regmap_reg_range(0x00E2, 0x00FE), /* NVM, interrupts, device ready */ + /* Update bits for synth config */ + regmap_reg_range(SI5341_SYNTH_N_UPD(0), SI5341_SYNTH_N_UPD(0)), + regmap_reg_range(SI5341_SYNTH_N_UPD(1), SI5341_SYNTH_N_UPD(1)), + regmap_reg_range(SI5341_SYNTH_N_UPD(2), SI5341_SYNTH_N_UPD(2)), + regmap_reg_range(SI5341_SYNTH_N_UPD(3), SI5341_SYNTH_N_UPD(3)), + regmap_reg_range(SI5341_SYNTH_N_UPD(4), SI5341_SYNTH_N_UPD(4)), +}; + +static const struct regmap_access_table si5341_regmap_volatile = { + .yes_ranges = si5341_regmap_volatile_range, + .n_yes_ranges = ARRAY_SIZE(si5341_regmap_volatile_range), +}; + +/* Pages 0, 1, 2, 3, 9, A, B are valid, so there are 12 pages */ +static const struct regmap_range_cfg si5341_regmap_ranges[] = { + { + .range_min = 0, + .range_max = SI5341_REGISTER_MAX, + .selector_reg = SI5341_PAGE, + .selector_mask = 0xff, + .selector_shift = 0, + .window_start = 0, + .window_len = 256, + }, +}; + +static const struct regmap_config si5341_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .cache_type = REGCACHE_RBTREE, + .ranges = si5341_regmap_ranges, + .num_ranges = ARRAY_SIZE(si5341_regmap_ranges), + .max_register = SI5341_REGISTER_MAX, + .volatile_table = &si5341_regmap_volatile, +}; + +static int si5341_dt_parse_dt(struct i2c_client *client, + struct clk_si5341_output_config *config) +{ + struct device_node *child; + struct device_node *np = client->dev.of_node; + u32 num; + u32 val; + + memset(config, 0, sizeof(struct clk_si5341_output_config) * + SI5341_MAX_NUM_OUTPUTS); + + for_each_child_of_node(np, child) { + if (of_property_read_u32(child, "reg", &num)) { + dev_err(&client->dev, "missing reg property of %s\n", + child->name); + goto put_child; + } + + if (num >= SI5341_MAX_NUM_OUTPUTS) { + dev_err(&client->dev, "invalid clkout %d\n", num); + goto put_child; + } + + if (!of_property_read_u32(child, "silabs,format", &val)) { + /* Set cm and ampl conservatively to 3v3 settings */ + switch (val) { + case 1: /* normal differential */ + config[num].out_cm_ampl_bits = 0x33; + break; + case 2: /* low-power differential */ + config[num].out_cm_ampl_bits = 0x13; + break; + case 4: /* LVCMOS */ + config[num].out_cm_ampl_bits = 0x33; + /* Set SI recommended impedance for LVCMOS */ + config[num].out_format_drv_bits |= 0xc0; + break; + default: + dev_err(&client->dev, + "invalid silabs,format %u for %u\n", + val, num); + goto put_child; + } + config[num].out_format_drv_bits &= ~0x07; + config[num].out_format_drv_bits |= val & 0x07; + /* Always enable the SYNC feature */ + config[num].out_format_drv_bits |= 0x08; + } + + if (!of_property_read_u32(child, "silabs,common-mode", &val)) { + if (val > 0xf) { + dev_err(&client->dev, + "invalid silabs,common-mode %u\n", + val); + goto put_child; + } + config[num].out_cm_ampl_bits &= 0xf0; + config[num].out_cm_ampl_bits |= val & 0x0f; + } + + if (!of_property_read_u32(child, "silabs,amplitude", &val)) { + if (val > 0xf) { + dev_err(&client->dev, + "invalid silabs,amplitude %u\n", + val); + goto put_child; + } + config[num].out_cm_ampl_bits &= 0x0f; + config[num].out_cm_ampl_bits |= (val << 4) & 0xf0; + } + + if (of_property_read_bool(child, "silabs,disable-high")) + config[num].out_format_drv_bits |= 0x10; + + config[num].synth_master = + of_property_read_bool(child, "silabs,synth-master"); + + config[num].always_on = + of_property_read_bool(child, "always-on"); + } + + return 0; + +put_child: + of_node_put(child); + return -EINVAL; +} + +/* + * If not pre-configured, calculate and set the PLL configuration manually. + * For low-jitter performance, the PLL should be set such that the synthesizers + * only need integer division. + * Without any user guidance, we'll set the PLL to 14GHz, which still allows + * the chip to generate any frequency on its outputs, but jitter performance + * may be sub-optimal. + */ +static int si5341_initialize_pll(struct clk_si5341 *data) +{ + struct device_node *np = data->i2c_client->dev.of_node; + u32 m_num = 0; + u32 m_den = 0; + + if (of_property_read_u32(np, "silabs,pll-m-num", &m_num)) { + dev_err(&data->i2c_client->dev, + "PLL configuration requires silabs,pll-m-num\n"); + } + if (of_property_read_u32(np, "silabs,pll-m-den", &m_den)) { + dev_err(&data->i2c_client->dev, + "PLL configuration requires silabs,pll-m-den\n"); + } + + if (!m_num || !m_den) { + dev_err(&data->i2c_client->dev, + "PLL configuration invalid, assume 14GHz\n"); + m_den = clk_get_rate(data->pxtal) / 10; + m_num = 1400000000; + } + + return si5341_encode_44_32(data->regmap, + SI5341_PLL_M_NUM, m_num, m_den); +} + +static int si5341_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct clk_si5341 *data; + struct clk_init_data init; + const char *root_clock_name; + const char *synth_clock_names[SI5341_NUM_SYNTH]; + int err; + unsigned int i; + struct clk_si5341_output_config config[SI5341_MAX_NUM_OUTPUTS]; + bool initialization_required; + + data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->i2c_client = client; + + data->pxtal = devm_clk_get(&client->dev, "xtal"); + if (IS_ERR(data->pxtal)) { + if (PTR_ERR(data->pxtal) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + dev_err(&client->dev, "Missing xtal clock input\n"); + } + + err = si5341_dt_parse_dt(client, config); + if (err) + return err; + + if (of_property_read_string(client->dev.of_node, "clock-output-names", + &init.name)) + init.name = client->dev.of_node->name; + root_clock_name = init.name; + + data->regmap = devm_regmap_init_i2c(client, &si5341_regmap_config); + if (IS_ERR(data->regmap)) + return PTR_ERR(data->regmap); + + i2c_set_clientdata(client, data); + + err = si5341_probe_chip_id(data); + if (err < 0) + return err; + + /* "Activate" the xtal (usually a fixed clock) */ + clk_prepare_enable(data->pxtal); + + if (of_property_read_bool(client->dev.of_node, "silabs,reprogram")) { + initialization_required = true; + } else { + err = si5341_is_programmed_already(data); + if (err < 0) + return err; + + initialization_required = !err; + } + + if (initialization_required) { + /* Populate the regmap cache in preparation for "cache only" */ + err = si5341_read_settings(data); + if (err < 0) + return err; + + err = si5341_send_preamble(data); + if (err < 0) + return err; + + /* + * We intend to send all 'final' register values in a single + * transaction. So cache all register writes until we're done + * configuring. + */ + regcache_cache_only(data->regmap, true); + + /* Write the configuration pairs from the firmware blob */ + err = si5341_write_multiple(data, si5341_reg_defaults, + ARRAY_SIZE(si5341_reg_defaults)); + if (err < 0) + return err; + + /* PLL configuration is required */ + err = si5341_initialize_pll(data); + if (err < 0) + return err; + } + + /* Register the PLL */ + data->pxtal_name = __clk_get_name(data->pxtal); + init.parent_names = &data->pxtal_name; + init.num_parents = 1; /* For now, only XTAL input supported */ + init.ops = &si5341_clk_ops; + init.flags = 0; + data->hw.init = &init; + + err = devm_clk_hw_register(&client->dev, &data->hw); + if (err) { + dev_err(&client->dev, "clock registration failed\n"); + return err; + } + + init.num_parents = 1; + init.parent_names = &root_clock_name; + init.ops = &si5341_synth_clk_ops; + for (i = 0; i < data->num_synth; ++i) { + synth_clock_names[i] = devm_kasprintf(&client->dev, GFP_KERNEL, + "%s.N%u", client->dev.of_node->name, i); + init.name = synth_clock_names[i]; + data->synth[i].index = i; + data->synth[i].data = data; + data->synth[i].hw.init = &init; + err = devm_clk_hw_register(&client->dev, &data->synth[i].hw); + if (err) { + dev_err(&client->dev, + "synth N%u registration failed\n", i); + } + } + + init.num_parents = data->num_synth; + init.parent_names = synth_clock_names; + init.ops = &si5341_output_clk_ops; + for (i = 0; i < data->num_outputs; ++i) { + init.name = kasprintf(GFP_KERNEL, "%s.%d", + client->dev.of_node->name, i); + init.flags = config[i].synth_master ? CLK_SET_RATE_PARENT : 0; + data->clk[i].index = i; + data->clk[i].data = data; + data->clk[i].hw.init = &init; + if (config[i].out_format_drv_bits & 0x07) { + regmap_write(data->regmap, + SI5341_OUT_FORMAT(&data->clk[i]), + config[i].out_format_drv_bits); + regmap_write(data->regmap, + SI5341_OUT_CM(&data->clk[i]), + config[i].out_cm_ampl_bits); + } + err = devm_clk_hw_register(&client->dev, &data->clk[i].hw); + kfree(init.name); /* clock framework made a copy of the name */ + if (err) { + dev_err(&client->dev, + "output %u registration failed\n", i); + return err; + } + if (config[i].always_on) + clk_prepare(data->clk[i].hw.clk); + } + + err = of_clk_add_hw_provider(client->dev.of_node, of_clk_si5341_get, + data); + if (err) { + dev_err(&client->dev, "unable to add clk provider\n"); + return err; + } + + if (initialization_required) { + /* Synchronize */ + regcache_cache_only(data->regmap, false); + err = regcache_sync(data->regmap); + if (err < 0) + return err; + + err = si5341_finalize_defaults(data); + if (err < 0) + return err; + } + + /* Free the names, clk framework makes copies */ + for (i = 0; i < data->num_synth; ++i) + devm_kfree(&client->dev, (void *)synth_clock_names[i]); + + return 0; +} + +static const struct i2c_device_id si5341_id[] = { + { "si5340", 0 }, + { "si5341", 1 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, si5341_id); + +static const struct of_device_id clk_si5341_of_match[] = { + { .compatible = "silabs,si5340" }, + { .compatible = "silabs,si5341" }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_si5341_of_match); + +static struct i2c_driver si5341_driver = { + .driver = { + .name = "si5341", + .of_match_table = clk_si5341_of_match, + }, + .probe = si5341_probe, + .id_table = si5341_id, +}; +module_i2c_driver(si5341_driver); + +MODULE_AUTHOR("Mike Looijmans <mike.looijmans@topic.nl>"); +MODULE_DESCRIPTION("Si5341 driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/clk-si544.c b/drivers/clk/clk-si544.c index 64e607f3232a..d9ec9086184d 100644 --- a/drivers/clk/clk-si544.c +++ b/drivers/clk/clk-si544.c @@ -7,6 +7,7 @@ #include <linux/clk-provider.h> #include <linux/delay.h> +#include <linux/math64.h> #include <linux/module.h> #include <linux/i2c.h> #include <linux/regmap.h> @@ -50,6 +51,11 @@ /* Lowest frequency synthesizeable using only the HS divider */ #define MIN_HSDIV_FREQ (FVCO_MIN / HS_DIV_MAX) +/* Range and interpretation of the adjustment value */ +#define DELTA_M_MAX 8161512 +#define DELTA_M_FRAC_NUM 19 +#define DELTA_M_FRAC_DEN 20000 + enum si544_speed_grade { si544a, si544b, @@ -71,12 +77,14 @@ struct clk_si544 { * @hs_div: 1st divider, 5..2046, must be even when >33 * @ls_div_bits: 2nd divider, as 2^x, range 0..5 * If ls_div_bits is non-zero, hs_div must be even + * @delta_m: Frequency shift for small -950..+950 ppm changes, 24 bit */ struct clk_si544_muldiv { u32 fb_div_frac; u16 fb_div_int; u16 hs_div; u8 ls_div_bits; + s32 delta_m; }; /* Enables or disables the output driver */ @@ -134,9 +142,30 @@ static int si544_get_muldiv(struct clk_si544 *data, settings->fb_div_int = reg[4] | (reg[5] & 0x07) << 8; settings->fb_div_frac = reg[0] | reg[1] << 8 | reg[2] << 16 | reg[3] << 24; + + err = regmap_bulk_read(data->regmap, SI544_REG_ADPLL_DELTA_M0, reg, 3); + if (err) + return err; + + /* Interpret as 24-bit signed number */ + settings->delta_m = reg[0] << 8 | reg[1] << 16 | reg[2] << 24; + settings->delta_m >>= 8; + return 0; } +static int si544_set_delta_m(struct clk_si544 *data, s32 delta_m) +{ + u8 reg[3]; + + reg[0] = delta_m; + reg[1] = delta_m >> 8; + reg[2] = delta_m >> 16; + + return regmap_bulk_write(data->regmap, SI544_REG_ADPLL_DELTA_M0, + reg, 3); +} + static int si544_set_muldiv(struct clk_si544 *data, struct clk_si544_muldiv *settings) { @@ -238,11 +267,15 @@ static int si544_calc_muldiv(struct clk_si544_muldiv *settings, do_div(vco, FXO); settings->fb_div_frac = vco; + /* Reset the frequency adjustment */ + settings->delta_m = 0; + return 0; } /* Calculate resulting frequency given the register settings */ -static unsigned long si544_calc_rate(struct clk_si544_muldiv *settings) +static unsigned long si544_calc_center_rate( + const struct clk_si544_muldiv *settings) { u32 d = settings->hs_div * BIT(settings->ls_div_bits); u64 vco; @@ -261,6 +294,25 @@ static unsigned long si544_calc_rate(struct clk_si544_muldiv *settings) return vco; } +static unsigned long si544_calc_rate(const struct clk_si544_muldiv *settings) +{ + unsigned long rate = si544_calc_center_rate(settings); + s64 delta = (s64)rate * (DELTA_M_FRAC_NUM * settings->delta_m); + + /* + * The clock adjustment is much smaller than 1 Hz, round to the + * nearest multiple. Apparently div64_s64 rounds towards zero, hence + * check the sign and adjust into the proper direction. + */ + if (settings->delta_m < 0) + delta -= ((s64)DELTA_M_MAX * DELTA_M_FRAC_DEN) / 2; + else + delta += ((s64)DELTA_M_MAX * DELTA_M_FRAC_DEN) / 2; + delta = div64_s64(delta, ((s64)DELTA_M_MAX * DELTA_M_FRAC_DEN)); + + return rate + delta; +} + static unsigned long si544_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { @@ -279,33 +331,60 @@ static long si544_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *parent_rate) { struct clk_si544 *data = to_clk_si544(hw); - struct clk_si544_muldiv settings; - int err; if (!is_valid_frequency(data, rate)) return -EINVAL; - err = si544_calc_muldiv(&settings, rate); - if (err) - return err; + /* The accuracy is less than 1 Hz, so any rate is possible */ + return rate; +} - return si544_calc_rate(&settings); +/* Calculates the maximum "small" change, 950 * rate / 1000000 */ +static unsigned long si544_max_delta(unsigned long rate) +{ + u64 num = rate; + + num *= DELTA_M_FRAC_NUM; + do_div(num, DELTA_M_FRAC_DEN); + + return num; +} + +static s32 si544_calc_delta(s32 delta, s32 max_delta) +{ + s64 n = (s64)delta * DELTA_M_MAX; + + return div_s64(n, max_delta); } -/* - * Update output frequency for "big" frequency changes - */ static int si544_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { struct clk_si544 *data = to_clk_si544(hw); struct clk_si544_muldiv settings; + unsigned long center; + long max_delta; + long delta; unsigned int old_oe_state; int err; if (!is_valid_frequency(data, rate)) return -EINVAL; + /* Try using the frequency adjustment feature for a <= 950ppm change */ + err = si544_get_muldiv(data, &settings); + if (err) + return err; + + center = si544_calc_center_rate(&settings); + max_delta = si544_max_delta(center); + delta = rate - center; + + if (abs(delta) <= max_delta) + return si544_set_delta_m(data, + si544_calc_delta(delta, max_delta)); + + /* Too big for the delta adjustment, need to reprogram */ err = si544_calc_muldiv(&settings, rate); if (err) return err; @@ -321,6 +400,9 @@ static int si544_set_rate(struct clk_hw *hw, unsigned long rate, if (err < 0) return err; + err = si544_set_delta_m(data, settings.delta_m); + if (err < 0) + return err; err = si544_set_muldiv(data, &settings); if (err < 0) diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 87b410d6e51d..c0990703ce54 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -1324,10 +1324,7 @@ static void clk_core_init_rate_req(struct clk_core * const core, static bool clk_core_can_round(struct clk_core * const core) { - if (core->ops->determine_rate || core->ops->round_rate) - return true; - - return false; + return core->ops->determine_rate || core->ops->round_rate; } static int clk_core_round_rate_nolock(struct clk_core *core, @@ -2194,7 +2191,7 @@ int clk_set_rate(struct clk *clk, unsigned long rate) EXPORT_SYMBOL_GPL(clk_set_rate); /** - * clk_set_rate_exclusive - specify a new rate get exclusive control + * clk_set_rate_exclusive - specify a new rate and get exclusive control * @clk: the clk whose rate is being changed * @rate: the new rate for clk * @@ -2202,7 +2199,7 @@ EXPORT_SYMBOL_GPL(clk_set_rate); * within a critical section * * This can be used initially to ensure that at least 1 consumer is - * statisfied when several consumers are competing for exclusivity over the + * satisfied when several consumers are competing for exclusivity over the * same clock provider. * * The exclusivity is not applied if setting the rate failed. @@ -2997,20 +2994,65 @@ static int clk_flags_show(struct seq_file *s, void *data) } DEFINE_SHOW_ATTRIBUTE(clk_flags); +static void possible_parent_show(struct seq_file *s, struct clk_core *core, + unsigned int i, char terminator) +{ + struct clk_core *parent; + + /* + * Go through the following options to fetch a parent's name. + * + * 1. Fetch the registered parent clock and use its name + * 2. Use the global (fallback) name if specified + * 3. Use the local fw_name if provided + * 4. Fetch parent clock's clock-output-name if DT index was set + * + * This may still fail in some cases, such as when the parent is + * specified directly via a struct clk_hw pointer, but it isn't + * registered (yet). + */ + parent = clk_core_get_parent_by_index(core, i); + if (parent) + seq_printf(s, "%s", parent->name); + else if (core->parents[i].name) + seq_printf(s, "%s", core->parents[i].name); + else if (core->parents[i].fw_name) + seq_printf(s, "<%s>(fw)", core->parents[i].fw_name); + else if (core->parents[i].index >= 0) + seq_printf(s, "%s", + of_clk_get_parent_name(core->of_node, + core->parents[i].index)); + else + seq_puts(s, "(missing)"); + + seq_putc(s, terminator); +} + static int possible_parents_show(struct seq_file *s, void *data) { struct clk_core *core = s->private; int i; for (i = 0; i < core->num_parents - 1; i++) - seq_printf(s, "%s ", core->parents[i].name); + possible_parent_show(s, core, i, ' '); - seq_printf(s, "%s\n", core->parents[i].name); + possible_parent_show(s, core, i, '\n'); return 0; } DEFINE_SHOW_ATTRIBUTE(possible_parents); +static int current_parent_show(struct seq_file *s, void *data) +{ + struct clk_core *core = s->private; + + if (core->parent) + seq_printf(s, "%s\n", core->parent->name); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(current_parent); + static int clk_duty_cycle_show(struct seq_file *s, void *data) { struct clk_core *core = s->private; @@ -3043,6 +3085,10 @@ static void clk_debug_create_one(struct clk_core *core, struct dentry *pdentry) debugfs_create_file("clk_duty_cycle", 0444, root, core, &clk_duty_cycle_fops); + if (core->num_parents > 0) + debugfs_create_file("clk_parent", 0444, root, core, + ¤t_parent_fops); + if (core->num_parents > 1) debugfs_create_file("clk_possible_parents", 0444, root, core, &possible_parents_fops); @@ -4038,6 +4084,7 @@ struct of_clk_provider { void *data; }; +extern struct of_device_id __clk_of_table; static const struct of_device_id __clk_of_table_sentinel __used __section(__clk_of_table_end); diff --git a/drivers/clk/clk.h b/drivers/clk/clk.h index d8400d623b34..2d801900cad5 100644 --- a/drivers/clk/clk.h +++ b/drivers/clk/clk.h @@ -33,10 +33,6 @@ clk_hw_create_clk(struct device *dev, struct clk_hw *hw, const char *dev_id, { return (struct clk *)hw; } -static struct clk_hw *__clk_get_hw(struct clk *clk) -{ - return (struct clk_hw *)clk; -} static inline void __clk_put(struct clk *clk) { } #endif diff --git a/drivers/clk/imx/clk-busy.c b/drivers/clk/imx/clk-busy.c index 7b35a11238ca..25c863da32c7 100644 --- a/drivers/clk/imx/clk-busy.c +++ b/drivers/clk/imx/clk-busy.c @@ -72,13 +72,14 @@ static const struct clk_ops clk_busy_divider_ops = { .set_rate = clk_busy_divider_set_rate, }; -struct clk *imx_clk_busy_divider(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_busy_divider(const char *name, const char *parent_name, void __iomem *reg, u8 shift, u8 width, void __iomem *busy_reg, u8 busy_shift) { struct clk_busy_divider *busy; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; busy = kzalloc(sizeof(*busy), GFP_KERNEL); if (!busy) @@ -101,11 +102,15 @@ struct clk *imx_clk_busy_divider(const char *name, const char *parent_name, busy->div.hw.init = &init; - clk = clk_register(NULL, &busy->div.hw); - if (IS_ERR(clk)) + hw = &busy->div.hw; + + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(busy); + return ERR_PTR(ret); + } - return clk; + return hw; } struct clk_busy_mux { @@ -146,13 +151,14 @@ static const struct clk_ops clk_busy_mux_ops = { .set_parent = clk_busy_mux_set_parent, }; -struct clk *imx_clk_busy_mux(const char *name, void __iomem *reg, u8 shift, +struct clk_hw *imx_clk_hw_busy_mux(const char *name, void __iomem *reg, u8 shift, u8 width, void __iomem *busy_reg, u8 busy_shift, const char * const *parent_names, int num_parents) { struct clk_busy_mux *busy; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; busy = kzalloc(sizeof(*busy), GFP_KERNEL); if (!busy) @@ -175,9 +181,13 @@ struct clk *imx_clk_busy_mux(const char *name, void __iomem *reg, u8 shift, busy->mux.hw.init = &init; - clk = clk_register(NULL, &busy->mux.hw); - if (IS_ERR(clk)) + hw = &busy->mux.hw; + + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(busy); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-cpu.c b/drivers/clk/imx/clk-cpu.c index 00d026eb7891..cb182bec79ba 100644 --- a/drivers/clk/imx/clk-cpu.c +++ b/drivers/clk/imx/clk-cpu.c @@ -69,13 +69,14 @@ static const struct clk_ops clk_cpu_ops = { .set_rate = clk_cpu_set_rate, }; -struct clk *imx_clk_cpu(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name, struct clk *div, struct clk *mux, struct clk *pll, struct clk *step) { struct clk_cpu *cpu; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; cpu = kzalloc(sizeof(*cpu), GFP_KERNEL); if (!cpu) @@ -93,10 +94,13 @@ struct clk *imx_clk_cpu(const char *name, const char *parent_name, init.num_parents = 1; cpu->hw.init = &init; + hw = &cpu->hw; - clk = clk_register(NULL, &cpu->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(cpu); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-fixup-div.c b/drivers/clk/imx/clk-fixup-div.c index bab46c6fd3db..4b17b91504ed 100644 --- a/drivers/clk/imx/clk-fixup-div.c +++ b/drivers/clk/imx/clk-fixup-div.c @@ -85,13 +85,14 @@ static const struct clk_ops clk_fixup_div_ops = { .set_rate = clk_fixup_div_set_rate, }; -struct clk *imx_clk_fixup_divider(const char *name, const char *parent, +struct clk_hw *imx_clk_hw_fixup_divider(const char *name, const char *parent, void __iomem *reg, u8 shift, u8 width, void (*fixup)(u32 *val)) { struct clk_fixup_div *fixup_div; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; if (!fixup) return ERR_PTR(-EINVAL); @@ -114,9 +115,13 @@ struct clk *imx_clk_fixup_divider(const char *name, const char *parent, fixup_div->ops = &clk_divider_ops; fixup_div->fixup = fixup; - clk = clk_register(NULL, &fixup_div->divider.hw); - if (IS_ERR(clk)) + hw = &fixup_div->divider.hw; + + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(fixup_div); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-fixup-mux.c b/drivers/clk/imx/clk-fixup-mux.c index 1aa3e8d9abf3..b569d919c645 100644 --- a/drivers/clk/imx/clk-fixup-mux.c +++ b/drivers/clk/imx/clk-fixup-mux.c @@ -63,13 +63,14 @@ static const struct clk_ops clk_fixup_mux_ops = { .set_parent = clk_fixup_mux_set_parent, }; -struct clk *imx_clk_fixup_mux(const char *name, void __iomem *reg, +struct clk_hw *imx_clk_hw_fixup_mux(const char *name, void __iomem *reg, u8 shift, u8 width, const char * const *parents, int num_parents, void (*fixup)(u32 *val)) { struct clk_fixup_mux *fixup_mux; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; if (!fixup) return ERR_PTR(-EINVAL); @@ -92,9 +93,13 @@ struct clk *imx_clk_fixup_mux(const char *name, void __iomem *reg, fixup_mux->ops = &clk_mux_ops; fixup_mux->fixup = fixup; - clk = clk_register(NULL, &fixup_mux->mux.hw); - if (IS_ERR(clk)) + hw = &fixup_mux->mux.hw; + + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(fixup_mux); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-gate-exclusive.c b/drivers/clk/imx/clk-gate-exclusive.c index cffa4966568d..77342893bb71 100644 --- a/drivers/clk/imx/clk-gate-exclusive.c +++ b/drivers/clk/imx/clk-gate-exclusive.c @@ -55,13 +55,14 @@ static const struct clk_ops clk_gate_exclusive_ops = { .is_enabled = clk_gate_exclusive_is_enabled, }; -struct clk *imx_clk_gate_exclusive(const char *name, const char *parent, +struct clk_hw *imx_clk_hw_gate_exclusive(const char *name, const char *parent, void __iomem *reg, u8 shift, u32 exclusive_mask) { struct clk_gate_exclusive *exgate; struct clk_gate *gate; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; if (exclusive_mask == 0) return ERR_PTR(-EINVAL); @@ -83,9 +84,13 @@ struct clk *imx_clk_gate_exclusive(const char *name, const char *parent, gate->hw.init = &init; exgate->exclusive_mask = exclusive_mask; - clk = clk_register(NULL, &gate->hw); - if (IS_ERR(clk)) - kfree(exgate); + hw = &gate->hw; - return clk; + ret = clk_hw_register(NULL, hw); + if (ret) { + kfree(gate); + return ERR_PTR(ret); + } + + return hw; } diff --git a/drivers/clk/imx/clk-gate2.c b/drivers/clk/imx/clk-gate2.c index ec08fda547a3..7d44ce814806 100644 --- a/drivers/clk/imx/clk-gate2.c +++ b/drivers/clk/imx/clk-gate2.c @@ -122,15 +122,16 @@ static const struct clk_ops clk_gate2_ops = { .is_enabled = clk_gate2_is_enabled, }; -struct clk *clk_register_gate2(struct device *dev, const char *name, +struct clk_hw *clk_hw_register_gate2(struct device *dev, const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 bit_idx, u8 cgr_val, u8 clk_gate2_flags, spinlock_t *lock, unsigned int *share_count) { struct clk_gate2 *gate; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; gate = kzalloc(sizeof(struct clk_gate2), GFP_KERNEL); if (!gate) @@ -151,10 +152,13 @@ struct clk *clk_register_gate2(struct device *dev, const char *name, init.num_parents = parent_name ? 1 : 0; gate->hw.init = &init; + hw = &gate->hw; - clk = clk_register(dev, &gate->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(gate); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index d5de733f336e..60f2de851f39 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -8,6 +8,7 @@ #include <linux/types.h> #include <linux/clk.h> #include <linux/clkdev.h> +#include <linux/clk-provider.h> #include <linux/err.h> #include <linux/io.h> #include <linux/of.h> @@ -87,8 +88,8 @@ static const char *pll5_bypass_sels[] = { "pll5", "pll5_bypass_src", }; static const char *pll6_bypass_sels[] = { "pll6", "pll6_bypass_src", }; static const char *pll7_bypass_sels[] = { "pll7", "pll7_bypass_src", }; -static struct clk *clk[IMX6QDL_CLK_END]; -static struct clk_onecell_data clk_data; +static struct clk_hw **hws; +static struct clk_hw_onecell_data *clk_hw_data; static struct clk_div_table clk_enet_ref_table[] = { { .val = 0, .div = 20, }, @@ -138,12 +139,13 @@ static inline int clk_on_imx6dl(void) return of_machine_is_compatible("fsl,imx6dl"); } -static struct clk ** const uart_clks[] __initconst = { - &clk[IMX6QDL_CLK_UART_IPG], - &clk[IMX6QDL_CLK_UART_SERIAL], - NULL +static const int uart_clk_ids[] __initconst = { + IMX6QDL_CLK_UART_IPG, + IMX6QDL_CLK_UART_SERIAL, }; +static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; + static int ldb_di_sel_by_clock_id(int clock_id) { switch (clock_id) { @@ -254,25 +256,14 @@ static bool pll6_bypassed(struct device_node *node) return false; } -#define CCM_CCDR 0x04 #define CCM_CCSR 0x0c #define CCM_CS2CDR 0x2c -#define CCDR_MMDC_CH1_MASK BIT(16) #define CCSR_PLL3_SW_CLK_SEL BIT(0) #define CS2CDR_LDB_DI0_CLK_SEL_SHIFT 9 #define CS2CDR_LDB_DI1_CLK_SEL_SHIFT 12 -static void __init imx6q_mmdc_ch1_mask_handshake(void __iomem *ccm_base) -{ - unsigned int reg; - - reg = readl_relaxed(ccm_base + CCM_CCDR); - reg |= CCDR_MMDC_CH1_MASK; - writel_relaxed(reg, ccm_base + CCM_CCDR); -} - /* * The only way to disable the MMDC_CH1 clock is to move it to pll3_sw_clk * via periph2_clk2_sel and then to disable pll3_sw_clk by selecting the @@ -282,14 +273,8 @@ static void mmdc_ch1_disable(void __iomem *ccm_base) { unsigned int reg; - clk_set_parent(clk[IMX6QDL_CLK_PERIPH2_CLK2_SEL], - clk[IMX6QDL_CLK_PLL3_USB_OTG]); - - /* - * Handshake with mmdc_ch1 module must be masked when changing - * periph2_clk_sel. - */ - clk_set_parent(clk[IMX6QDL_CLK_PERIPH2], clk[IMX6QDL_CLK_PERIPH2_CLK2]); + clk_set_parent(hws[IMX6QDL_CLK_PERIPH2_CLK2_SEL]->clk, + hws[IMX6QDL_CLK_PLL3_USB_OTG]->clk); /* Disable pll3_sw_clk by selecting the bypass clock source */ reg = readl_relaxed(ccm_base + CCM_CCSR); @@ -305,8 +290,6 @@ static void mmdc_ch1_reenable(void __iomem *ccm_base) reg = readl_relaxed(ccm_base + CCM_CCSR); reg &= ~CCSR_PLL3_SW_CLK_SEL; writel_relaxed(reg, ccm_base + CCM_CCSR); - - clk_set_parent(clk[IMX6QDL_CLK_PERIPH2], clk[IMX6QDL_CLK_PERIPH2_PRE]); } /* @@ -365,8 +348,8 @@ static void init_ldb_clks(struct device_node *np, void __iomem *ccm_base) /* Only switch to or from pll2_pfd2_396m if it is disabled */ if ((sel[i][0] == 2 || sel[i][3] == 2) && - (clk_get_parent(clk[IMX6QDL_CLK_PERIPH_PRE]) == - clk[IMX6QDL_CLK_PLL2_PFD2_396M])) { + (clk_get_parent(hws[IMX6QDL_CLK_PERIPH_PRE]->clk) == + hws[IMX6QDL_CLK_PLL2_PFD2_396M]->clk)) { pr_err("ccm: ldb_di%d_sel: couldn't disable pll2_pfd2_396m\n", i); sel[i][3] = sel[i][2] = sel[i][1] = sel[i][0]; @@ -418,8 +401,8 @@ static void disable_anatop_clocks(void __iomem *anatop_base) /* Make sure PLL2 PFDs 0-2 are gated */ reg = readl_relaxed(anatop_base + CCM_ANALOG_PFD_528); /* Cannot gate PFD2 if pll2_pfd2_396m is the parent of MMDC clock */ - if (clk_get_parent(clk[IMX6QDL_CLK_PERIPH_PRE]) == - clk[IMX6QDL_CLK_PLL2_PFD2_396M]) + if (clk_get_parent(hws[IMX6QDL_CLK_PERIPH_PRE]->clk) == + hws[IMX6QDL_CLK_PLL2_PFD2_396M]->clk) reg |= PFD0_CLKGATE | PFD1_CLKGATE; else reg |= PFD0_CLKGATE | PFD1_CLKGATE | PFD2_CLKGATE; @@ -436,31 +419,45 @@ static void disable_anatop_clocks(void __iomem *anatop_base) writel_relaxed(reg, anatop_base + CCM_ANALOG_PLL_VIDEO); } +static struct clk_hw * __init imx6q_obtain_fixed_clk_hw(struct device_node *np, + const char *name, + unsigned long rate) +{ + struct clk *clk = of_clk_get_by_name(np, name); + struct clk_hw *hw; + + if (IS_ERR(clk)) + hw = imx_obtain_fixed_clock_hw(name, rate); + else + hw = __clk_get_hw(clk); + + return hw; +} + static void __init imx6q_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *anatop_base, *base; int ret; + int i; + + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX6QDL_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return; - clk[IMX6QDL_CLK_DUMMY] = imx_clk_fixed("dummy", 0); - clk[IMX6QDL_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil"); - if (IS_ERR(clk[IMX6QDL_CLK_CKIL])) - clk[IMX6QDL_CLK_CKIL] = imx_obtain_fixed_clock("ckil", 0); - clk[IMX6QDL_CLK_CKIH] = of_clk_get_by_name(ccm_node, "ckih1"); - if (IS_ERR(clk[IMX6QDL_CLK_CKIH])) - clk[IMX6QDL_CLK_CKIH] = imx_obtain_fixed_clock("ckih1", 0); - clk[IMX6QDL_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc"); - if (IS_ERR(clk[IMX6QDL_CLK_OSC])) - clk[IMX6QDL_CLK_OSC] = imx_obtain_fixed_clock("osc", 0); + clk_hw_data->num = IMX6QDL_CLK_END; + hws = clk_hw_data->hws; - /* Clock source from external clock via CLK1/2 PADs */ - clk[IMX6QDL_CLK_ANACLK1] = of_clk_get_by_name(ccm_node, "anaclk1"); - if (IS_ERR(clk[IMX6QDL_CLK_ANACLK1])) - clk[IMX6QDL_CLK_ANACLK1] = imx_obtain_fixed_clock("anaclk1", 0); + hws[IMX6QDL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); - clk[IMX6QDL_CLK_ANACLK2] = of_clk_get_by_name(ccm_node, "anaclk2"); - if (IS_ERR(clk[IMX6QDL_CLK_ANACLK2])) - clk[IMX6QDL_CLK_ANACLK2] = imx_obtain_fixed_clock("anaclk2", 0); + hws[IMX6QDL_CLK_CKIL] = imx6q_obtain_fixed_clk_hw(ccm_node, "ckil", 0); + hws[IMX6QDL_CLK_CKIH] = imx6q_obtain_fixed_clk_hw(ccm_node, "ckih1", 0); + hws[IMX6QDL_CLK_OSC] = imx6q_obtain_fixed_clk_hw(ccm_node, "osc", 0); + + /* Clock source from external clock via CLK1/2 PADs */ + hws[IMX6QDL_CLK_ANACLK1] = imx6q_obtain_fixed_clk_hw(ccm_node, "anaclk1", 0); + hws[IMX6QDL_CLK_ANACLK2] = imx6q_obtain_fixed_clk_hw(ccm_node, "anaclk2", 0); np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-anatop"); anatop_base = base = of_iomap(np, 0); @@ -475,47 +472,47 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) video_div_table[3].div = 1; } - clk[IMX6QDL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clk[IMX6QDL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clk[IMX6QDL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clk[IMX6QDL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clk[IMX6QDL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clk[IMX6QDL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clk[IMX6QDL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6QDL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6QDL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6QDL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6QDL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6QDL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6QDL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6QDL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); /* type name parent_name base div_mask */ - clk[IMX6QDL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); - clk[IMX6QDL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); - clk[IMX6QDL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); - clk[IMX6QDL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); - clk[IMX6QDL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); - clk[IMX6QDL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); - clk[IMX6QDL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); - - clk[IMX6QDL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + hws[IMX6QDL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + hws[IMX6QDL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + hws[IMX6QDL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + hws[IMX6QDL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + hws[IMX6QDL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + hws[IMX6QDL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); + + hws[IMX6QDL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); /* Do not bypass PLLs initially */ - clk_set_parent(clk[IMX6QDL_PLL1_BYPASS], clk[IMX6QDL_CLK_PLL1]); - clk_set_parent(clk[IMX6QDL_PLL2_BYPASS], clk[IMX6QDL_CLK_PLL2]); - clk_set_parent(clk[IMX6QDL_PLL3_BYPASS], clk[IMX6QDL_CLK_PLL3]); - clk_set_parent(clk[IMX6QDL_PLL4_BYPASS], clk[IMX6QDL_CLK_PLL4]); - clk_set_parent(clk[IMX6QDL_PLL5_BYPASS], clk[IMX6QDL_CLK_PLL5]); - clk_set_parent(clk[IMX6QDL_PLL6_BYPASS], clk[IMX6QDL_CLK_PLL6]); - clk_set_parent(clk[IMX6QDL_PLL7_BYPASS], clk[IMX6QDL_CLK_PLL7]); - - clk[IMX6QDL_CLK_PLL1_SYS] = imx_clk_gate("pll1_sys", "pll1_bypass", base + 0x00, 13); - clk[IMX6QDL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); - clk[IMX6QDL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); - clk[IMX6QDL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); - clk[IMX6QDL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); - clk[IMX6QDL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); - clk[IMX6QDL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); + clk_set_parent(hws[IMX6QDL_PLL1_BYPASS]->clk, hws[IMX6QDL_CLK_PLL1]->clk); + clk_set_parent(hws[IMX6QDL_PLL2_BYPASS]->clk, hws[IMX6QDL_CLK_PLL2]->clk); + clk_set_parent(hws[IMX6QDL_PLL3_BYPASS]->clk, hws[IMX6QDL_CLK_PLL3]->clk); + clk_set_parent(hws[IMX6QDL_PLL4_BYPASS]->clk, hws[IMX6QDL_CLK_PLL4]->clk); + clk_set_parent(hws[IMX6QDL_PLL5_BYPASS]->clk, hws[IMX6QDL_CLK_PLL5]->clk); + clk_set_parent(hws[IMX6QDL_PLL6_BYPASS]->clk, hws[IMX6QDL_CLK_PLL6]->clk); + clk_set_parent(hws[IMX6QDL_PLL7_BYPASS]->clk, hws[IMX6QDL_CLK_PLL7]->clk); + + hws[IMX6QDL_CLK_PLL1_SYS] = imx_clk_hw_gate("pll1_sys", "pll1_bypass", base + 0x00, 13); + hws[IMX6QDL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); + hws[IMX6QDL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); + hws[IMX6QDL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); + hws[IMX6QDL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); + hws[IMX6QDL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); + hws[IMX6QDL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); /* * Bit 20 is the reserved and read-only bit, we do this only for: @@ -523,15 +520,15 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) * - Keep refcount when do usbphy clk_enable/disable, in that case, * the clk framework may need to enable/disable usbphy's parent */ - clk[IMX6QDL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); - clk[IMX6QDL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); + hws[IMX6QDL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); + hws[IMX6QDL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); /* * usbphy*_gate needs to be on after system boots up, and software * never needs to control it anymore. */ - clk[IMX6QDL_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6); - clk[IMX6QDL_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6); + hws[IMX6QDL_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6); + hws[IMX6QDL_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6); /* * The ENET PLL is special in that is has multiple outputs with @@ -545,22 +542,22 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) * */ if (!pll6_bypassed(ccm_node)) { - clk[IMX6QDL_CLK_SATA_REF] = imx_clk_fixed_factor("sata_ref", "pll6_enet", 1, 5); - clk[IMX6QDL_CLK_PCIE_REF] = imx_clk_fixed_factor("pcie_ref", "pll6_enet", 1, 4); - clk[IMX6QDL_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, + hws[IMX6QDL_CLK_SATA_REF] = imx_clk_hw_fixed_factor("sata_ref", "pll6_enet", 1, 5); + hws[IMX6QDL_CLK_PCIE_REF] = imx_clk_hw_fixed_factor("pcie_ref", "pll6_enet", 1, 4); + hws[IMX6QDL_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock); } else { - clk[IMX6QDL_CLK_SATA_REF] = imx_clk_fixed_factor("sata_ref", "pll6_enet", 1, 1); - clk[IMX6QDL_CLK_PCIE_REF] = imx_clk_fixed_factor("pcie_ref", "pll6_enet", 1, 1); - clk[IMX6QDL_CLK_ENET_REF] = imx_clk_fixed_factor("enet_ref", "pll6_enet", 1, 1); + hws[IMX6QDL_CLK_SATA_REF] = imx_clk_hw_fixed_factor("sata_ref", "pll6_enet", 1, 1); + hws[IMX6QDL_CLK_PCIE_REF] = imx_clk_hw_fixed_factor("pcie_ref", "pll6_enet", 1, 1); + hws[IMX6QDL_CLK_ENET_REF] = imx_clk_hw_fixed_factor("enet_ref", "pll6_enet", 1, 1); } - clk[IMX6QDL_CLK_SATA_REF_100M] = imx_clk_gate("sata_ref_100m", "sata_ref", base + 0xe0, 20); - clk[IMX6QDL_CLK_PCIE_REF_125M] = imx_clk_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19); + hws[IMX6QDL_CLK_SATA_REF_100M] = imx_clk_hw_gate("sata_ref_100m", "sata_ref", base + 0xe0, 20); + hws[IMX6QDL_CLK_PCIE_REF_125M] = imx_clk_hw_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19); - clk[IMX6QDL_CLK_LVDS1_SEL] = imx_clk_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); - clk[IMX6QDL_CLK_LVDS2_SEL] = imx_clk_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); + hws[IMX6QDL_CLK_LVDS1_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); + hws[IMX6QDL_CLK_LVDS2_SEL] = imx_clk_hw_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); /* * lvds1_gate and lvds2_gate are pseudo-gates. Both can be @@ -572,84 +569,84 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) * it. */ writel(readl(base + 0x160) & ~0x3c00, base + 0x160); - clk[IMX6QDL_CLK_LVDS1_GATE] = imx_clk_gate_exclusive("lvds1_gate", "lvds1_sel", base + 0x160, 10, BIT(12)); - clk[IMX6QDL_CLK_LVDS2_GATE] = imx_clk_gate_exclusive("lvds2_gate", "lvds2_sel", base + 0x160, 11, BIT(13)); + hws[IMX6QDL_CLK_LVDS1_GATE] = imx_clk_hw_gate_exclusive("lvds1_gate", "lvds1_sel", base + 0x160, 10, BIT(12)); + hws[IMX6QDL_CLK_LVDS2_GATE] = imx_clk_hw_gate_exclusive("lvds2_gate", "lvds2_sel", base + 0x160, 11, BIT(13)); - clk[IMX6QDL_CLK_LVDS1_IN] = imx_clk_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10)); - clk[IMX6QDL_CLK_LVDS2_IN] = imx_clk_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11)); + hws[IMX6QDL_CLK_LVDS1_IN] = imx_clk_hw_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10)); + hws[IMX6QDL_CLK_LVDS2_IN] = imx_clk_hw_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11)); /* name parent_name reg idx */ - clk[IMX6QDL_CLK_PLL2_PFD0_352M] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); - clk[IMX6QDL_CLK_PLL2_PFD1_594M] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); - clk[IMX6QDL_CLK_PLL2_PFD2_396M] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); - clk[IMX6QDL_CLK_PLL3_PFD0_720M] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); - clk[IMX6QDL_CLK_PLL3_PFD1_540M] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); - clk[IMX6QDL_CLK_PLL3_PFD2_508M] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); - clk[IMX6QDL_CLK_PLL3_PFD3_454M] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); + hws[IMX6QDL_CLK_PLL2_PFD0_352M] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); + hws[IMX6QDL_CLK_PLL2_PFD1_594M] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); + hws[IMX6QDL_CLK_PLL2_PFD2_396M] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); + hws[IMX6QDL_CLK_PLL3_PFD0_720M] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); + hws[IMX6QDL_CLK_PLL3_PFD1_540M] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); + hws[IMX6QDL_CLK_PLL3_PFD2_508M] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); + hws[IMX6QDL_CLK_PLL3_PFD3_454M] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); /* name parent_name mult div */ - clk[IMX6QDL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); - clk[IMX6QDL_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); - clk[IMX6QDL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); - clk[IMX6QDL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); - clk[IMX6QDL_CLK_TWD] = imx_clk_fixed_factor("twd", "arm", 1, 2); - clk[IMX6QDL_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8); - clk[IMX6QDL_CLK_VIDEO_27M] = imx_clk_fixed_factor("video_27m", "pll3_pfd1_540m", 1, 20); + hws[IMX6QDL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); + hws[IMX6QDL_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); + hws[IMX6QDL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); + hws[IMX6QDL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); + hws[IMX6QDL_CLK_TWD] = imx_clk_hw_fixed_factor("twd", "arm", 1, 2); + hws[IMX6QDL_CLK_GPT_3M] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8); + hws[IMX6QDL_CLK_VIDEO_27M] = imx_clk_hw_fixed_factor("video_27m", "pll3_pfd1_540m", 1, 20); if (clk_on_imx6dl() || clk_on_imx6qp()) { - clk[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_fixed_factor("gpu2d_axi", "mmdc_ch0_axi_podf", 1, 1); - clk[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_fixed_factor("gpu3d_axi", "mmdc_ch0_axi_podf", 1, 1); + hws[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_hw_fixed_factor("gpu2d_axi", "mmdc_ch0_axi_podf", 1, 1); + hws[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_hw_fixed_factor("gpu3d_axi", "mmdc_ch0_axi_podf", 1, 1); } - clk[IMX6QDL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); - clk[IMX6QDL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock); - clk[IMX6QDL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); - clk[IMX6QDL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); + hws[IMX6QDL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); + hws[IMX6QDL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock); + hws[IMX6QDL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); + hws[IMX6QDL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); np = ccm_node; base = of_iomap(np, 0); WARN_ON(!base); /* name reg shift width parent_names num_parents */ - clk[IMX6QDL_CLK_STEP] = imx_clk_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels)); - clk[IMX6QDL_CLK_PLL1_SW] = imx_clk_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels)); - clk[IMX6QDL_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); - clk[IMX6QDL_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); - clk[IMX6QDL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); - clk[IMX6QDL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); - clk[IMX6QDL_CLK_AXI_SEL] = imx_clk_mux("axi_sel", base + 0x14, 6, 2, axi_sels, ARRAY_SIZE(axi_sels)); - clk[IMX6QDL_CLK_ESAI_SEL] = imx_clk_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clk[IMX6QDL_CLK_ASRC_SEL] = imx_clk_mux("asrc_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clk[IMX6QDL_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6QDL_CLK_STEP] = imx_clk_hw_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels)); + hws[IMX6QDL_CLK_PLL1_SW] = imx_clk_hw_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels)); + hws[IMX6QDL_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); + hws[IMX6QDL_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); + hws[IMX6QDL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); + hws[IMX6QDL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); + hws[IMX6QDL_CLK_AXI_SEL] = imx_clk_hw_mux("axi_sel", base + 0x14, 6, 2, axi_sels, ARRAY_SIZE(axi_sels)); + hws[IMX6QDL_CLK_ESAI_SEL] = imx_clk_hw_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6QDL_CLK_ASRC_SEL] = imx_clk_hw_mux("asrc_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6QDL_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels)); if (clk_on_imx6q()) { - clk[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_mux("gpu2d_axi", base + 0x18, 0, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels)); - clk[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_mux("gpu3d_axi", base + 0x18, 1, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels)); + hws[IMX6QDL_CLK_GPU2D_AXI] = imx_clk_hw_mux("gpu2d_axi", base + 0x18, 0, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels)); + hws[IMX6QDL_CLK_GPU3D_AXI] = imx_clk_hw_mux("gpu3d_axi", base + 0x18, 1, 1, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels)); } if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_CAN_SEL] = imx_clk_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels)); - clk[IMX6QDL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); - clk[IMX6QDL_CLK_IPG_PER_SEL] = imx_clk_mux("ipg_per_sel", base + 0x1c, 6, 1, ipg_per_sels, ARRAY_SIZE(ipg_per_sels)); - clk[IMX6QDL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); - clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels_2, ARRAY_SIZE(gpu2d_core_sels_2)); + hws[IMX6QDL_CLK_CAN_SEL] = imx_clk_hw_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels)); + hws[IMX6QDL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); + hws[IMX6QDL_CLK_IPG_PER_SEL] = imx_clk_hw_mux("ipg_per_sel", base + 0x1c, 6, 1, ipg_per_sels, ARRAY_SIZE(ipg_per_sels)); + hws[IMX6QDL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); + hws[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_hw_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels_2, ARRAY_SIZE(gpu2d_core_sels_2)); } else if (clk_on_imx6dl()) { - clk[IMX6QDL_CLK_MLB_SEL] = imx_clk_mux("mlb_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels)); + hws[IMX6QDL_CLK_MLB_SEL] = imx_clk_hw_mux("mlb_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels)); } else { - clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels)); + hws[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_hw_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels, ARRAY_SIZE(gpu2d_core_sels)); } - clk[IMX6QDL_CLK_GPU3D_CORE_SEL] = imx_clk_mux("gpu3d_core_sel", base + 0x18, 4, 2, gpu3d_core_sels, ARRAY_SIZE(gpu3d_core_sels)); + hws[IMX6QDL_CLK_GPU3D_CORE_SEL] = imx_clk_hw_mux("gpu3d_core_sel", base + 0x18, 4, 2, gpu3d_core_sels, ARRAY_SIZE(gpu3d_core_sels)); if (clk_on_imx6dl()) - clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels)); + hws[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_hw_mux("gpu2d_core_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels)); else - clk[IMX6QDL_CLK_GPU3D_SHADER_SEL] = imx_clk_mux("gpu3d_shader_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels)); - clk[IMX6QDL_CLK_IPU1_SEL] = imx_clk_mux("ipu1_sel", base + 0x3c, 9, 2, ipu_sels, ARRAY_SIZE(ipu_sels)); - clk[IMX6QDL_CLK_IPU2_SEL] = imx_clk_mux("ipu2_sel", base + 0x3c, 14, 2, ipu_sels, ARRAY_SIZE(ipu_sels)); + hws[IMX6QDL_CLK_GPU3D_SHADER_SEL] = imx_clk_hw_mux("gpu3d_shader_sel", base + 0x18, 8, 2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels)); + hws[IMX6QDL_CLK_IPU1_SEL] = imx_clk_hw_mux("ipu1_sel", base + 0x3c, 9, 2, ipu_sels, ARRAY_SIZE(ipu_sels)); + hws[IMX6QDL_CLK_IPU2_SEL] = imx_clk_hw_mux("ipu2_sel", base + 0x3c, 14, 2, ipu_sels, ARRAY_SIZE(ipu_sels)); disable_anatop_clocks(anatop_base); - imx6q_mmdc_ch1_mask_handshake(base); + imx_mmdc_mask_handshake(base, 1); if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_hw_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT); } else { /* * The LDB_DI0/1_SEL muxes are registered read-only due to a hardware @@ -658,322 +655,333 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) */ init_ldb_clks(np, base); - clk[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_mux_ldb("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels)); - clk[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_mux_ldb("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels)); + hws[IMX6QDL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux_ldb("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels)); + hws[IMX6QDL_CLK_LDB_DI1_SEL] = imx_clk_hw_mux_ldb("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels, ARRAY_SIZE(ldb_di_sels)); } - clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL] = imx_clk_mux_flags("ipu1_di0_pre_sel", base + 0x34, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL] = imx_clk_mux_flags("ipu1_di1_pre_sel", base + 0x34, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL] = imx_clk_mux_flags("ipu2_di0_pre_sel", base + 0x38, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU2_DI1_PRE_SEL] = imx_clk_mux_flags("ipu2_di1_pre_sel", base + 0x38, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_HSI_TX_SEL] = imx_clk_mux("hsi_tx_sel", base + 0x30, 28, 1, hsi_tx_sels, ARRAY_SIZE(hsi_tx_sels)); - clk[IMX6QDL_CLK_PCIE_AXI_SEL] = imx_clk_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels)); + + hws[IMX6QDL_CLK_IPU1_DI0_PRE_SEL] = imx_clk_hw_mux_flags("ipu1_di0_pre_sel", base + 0x34, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU1_DI1_PRE_SEL] = imx_clk_hw_mux_flags("ipu1_di1_pre_sel", base + 0x34, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU2_DI0_PRE_SEL] = imx_clk_hw_mux_flags("ipu2_di0_pre_sel", base + 0x38, 6, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU2_DI1_PRE_SEL] = imx_clk_hw_mux_flags("ipu2_di1_pre_sel", base + 0x38, 15, 3, ipu_di_pre_sels, ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_HSI_TX_SEL] = imx_clk_hw_mux("hsi_tx_sel", base + 0x30, 28, 1, hsi_tx_sels, ARRAY_SIZE(hsi_tx_sels)); + hws[IMX6QDL_CLK_PCIE_AXI_SEL] = imx_clk_hw_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels)); + if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels_2, ARRAY_SIZE(ipu1_di0_sels_2), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels_2, ARRAY_SIZE(ipu1_di1_sels_2), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels_2, ARRAY_SIZE(ipu2_di0_sels_2), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels_2, ARRAY_SIZE(ipu2_di1_sels_2), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_SSI1_SEL] = imx_clk_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clk[IMX6QDL_CLK_SSI2_SEL] = imx_clk_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clk[IMX6QDL_CLK_SSI3_SEL] = imx_clk_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clk[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clk[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clk[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clk[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clk[IMX6QDL_CLK_ENFC_SEL] = imx_clk_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels_2, ARRAY_SIZE(enfc_sels_2)); - clk[IMX6QDL_CLK_EIM_SEL] = imx_clk_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels)); - clk[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels)); - clk[IMX6QDL_CLK_PRE_AXI] = imx_clk_mux("pre_axi", base + 0x18, 1, 1, pre_axi_sels, ARRAY_SIZE(pre_axi_sels)); + hws[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_hw_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels_2, ARRAY_SIZE(ipu1_di0_sels_2), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_hw_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels_2, ARRAY_SIZE(ipu1_di1_sels_2), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_hw_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels_2, ARRAY_SIZE(ipu2_di0_sels_2), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_hw_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels_2, ARRAY_SIZE(ipu2_di1_sels_2), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_SSI1_SEL] = imx_clk_hw_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6QDL_CLK_SSI2_SEL] = imx_clk_hw_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6QDL_CLK_SSI3_SEL] = imx_clk_hw_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_hw_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_hw_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6QDL_CLK_ENFC_SEL] = imx_clk_hw_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels_2, ARRAY_SIZE(enfc_sels_2)); + hws[IMX6QDL_CLK_EIM_SEL] = imx_clk_hw_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels)); + hws[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_hw_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels)); + hws[IMX6QDL_CLK_PRE_AXI] = imx_clk_hw_mux("pre_axi", base + 0x18, 1, 1, pre_axi_sels, ARRAY_SIZE(pre_axi_sels)); } else { - clk[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels, ARRAY_SIZE(ipu1_di0_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels, ARRAY_SIZE(ipu1_di1_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels, ARRAY_SIZE(ipu2_di0_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels, ARRAY_SIZE(ipu2_di1_sels), CLK_SET_RATE_PARENT); - clk[IMX6QDL_CLK_SSI1_SEL] = imx_clk_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_SSI2_SEL] = imx_clk_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_SSI3_SEL] = imx_clk_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_ENFC_SEL] = imx_clk_mux("enfc_sel", base + 0x2c, 16, 2, enfc_sels, ARRAY_SIZE(enfc_sels)); - clk[IMX6QDL_CLK_EIM_SEL] = imx_clk_fixup_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels), imx_cscmr1_fixup); - clk[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_fixup_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_IPU1_DI0_SEL] = imx_clk_hw_mux_flags("ipu1_di0_sel", base + 0x34, 0, 3, ipu1_di0_sels, ARRAY_SIZE(ipu1_di0_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU1_DI1_SEL] = imx_clk_hw_mux_flags("ipu1_di1_sel", base + 0x34, 9, 3, ipu1_di1_sels, ARRAY_SIZE(ipu1_di1_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU2_DI0_SEL] = imx_clk_hw_mux_flags("ipu2_di0_sel", base + 0x38, 0, 3, ipu2_di0_sels, ARRAY_SIZE(ipu2_di0_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_IPU2_DI1_SEL] = imx_clk_hw_mux_flags("ipu2_di1_sel", base + 0x38, 9, 3, ipu2_di1_sels, ARRAY_SIZE(ipu2_di1_sels), CLK_SET_RATE_PARENT); + hws[IMX6QDL_CLK_SSI1_SEL] = imx_clk_hw_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_SSI2_SEL] = imx_clk_hw_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_SSI3_SEL] = imx_clk_hw_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_USDHC1_SEL] = imx_clk_hw_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_USDHC2_SEL] = imx_clk_hw_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_USDHC3_SEL] = imx_clk_hw_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_USDHC4_SEL] = imx_clk_hw_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_ENFC_SEL] = imx_clk_hw_mux("enfc_sel", base + 0x2c, 16, 2, enfc_sels, ARRAY_SIZE(enfc_sels)); + hws[IMX6QDL_CLK_EIM_SEL] = imx_clk_hw_fixup_mux("eim_sel", base + 0x1c, 27, 2, eim_sels, ARRAY_SIZE(eim_sels), imx_cscmr1_fixup); + hws[IMX6QDL_CLK_EIM_SLOW_SEL] = imx_clk_hw_fixup_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels), imx_cscmr1_fixup); } - clk[IMX6QDL_CLK_VDO_AXI_SEL] = imx_clk_mux("vdo_axi_sel", base + 0x18, 11, 1, vdo_axi_sels, ARRAY_SIZE(vdo_axi_sels)); - clk[IMX6QDL_CLK_VPU_AXI_SEL] = imx_clk_mux("vpu_axi_sel", base + 0x18, 14, 2, vpu_axi_sels, ARRAY_SIZE(vpu_axi_sels)); - clk[IMX6QDL_CLK_CKO1_SEL] = imx_clk_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels)); - clk[IMX6QDL_CLK_CKO2_SEL] = imx_clk_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels)); - clk[IMX6QDL_CLK_CKO] = imx_clk_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels)); + + hws[IMX6QDL_CLK_VDO_AXI_SEL] = imx_clk_hw_mux("vdo_axi_sel", base + 0x18, 11, 1, vdo_axi_sels, ARRAY_SIZE(vdo_axi_sels)); + hws[IMX6QDL_CLK_VPU_AXI_SEL] = imx_clk_hw_mux("vpu_axi_sel", base + 0x18, 14, 2, vpu_axi_sels, ARRAY_SIZE(vpu_axi_sels)); + hws[IMX6QDL_CLK_CKO1_SEL] = imx_clk_hw_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels)); + hws[IMX6QDL_CLK_CKO2_SEL] = imx_clk_hw_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels)); + hws[IMX6QDL_CLK_CKO] = imx_clk_hw_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels)); /* name reg shift width busy: reg, shift parent_names num_parents */ - clk[IMX6QDL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); - clk[IMX6QDL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); + hws[IMX6QDL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); + hws[IMX6QDL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); /* name parent_name reg shift width */ - clk[IMX6QDL_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); - clk[IMX6QDL_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); - clk[IMX6QDL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2); - clk[IMX6QDL_CLK_ESAI_PRED] = imx_clk_divider("esai_pred", "esai_sel", base + 0x28, 9, 3); - clk[IMX6QDL_CLK_ESAI_PODF] = imx_clk_divider("esai_podf", "esai_pred", base + 0x28, 25, 3); - clk[IMX6QDL_CLK_ASRC_PRED] = imx_clk_divider("asrc_pred", "asrc_sel", base + 0x30, 12, 3); - clk[IMX6QDL_CLK_ASRC_PODF] = imx_clk_divider("asrc_podf", "asrc_pred", base + 0x30, 9, 3); - clk[IMX6QDL_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); - clk[IMX6QDL_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); + hws[IMX6QDL_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); + hws[IMX6QDL_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); + hws[IMX6QDL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2); + hws[IMX6QDL_CLK_ESAI_PRED] = imx_clk_hw_divider("esai_pred", "esai_sel", base + 0x28, 9, 3); + hws[IMX6QDL_CLK_ESAI_PODF] = imx_clk_hw_divider("esai_podf", "esai_pred", base + 0x28, 25, 3); + hws[IMX6QDL_CLK_ASRC_PRED] = imx_clk_hw_divider("asrc_pred", "asrc_sel", base + 0x30, 12, 3); + hws[IMX6QDL_CLK_ASRC_PODF] = imx_clk_hw_divider("asrc_podf", "asrc_pred", base + 0x30, 9, 3); + hws[IMX6QDL_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); + hws[IMX6QDL_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); + if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_IPG_PER] = imx_clk_divider("ipg_per", "ipg_per_sel", base + 0x1c, 0, 6); - clk[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6); - clk[IMX6QDL_CLK_CAN_ROOT] = imx_clk_divider("can_root", "can_sel", base + 0x20, 2, 6); - clk[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_divider("uart_serial_podf", "uart_sel", base + 0x24, 0, 6); - clk[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0", 2, 7); - clk[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1", 2, 7); + hws[IMX6QDL_CLK_IPG_PER] = imx_clk_hw_divider("ipg_per", "ipg_per_sel", base + 0x1c, 0, 6); + hws[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_hw_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6); + hws[IMX6QDL_CLK_CAN_ROOT] = imx_clk_hw_divider("can_root", "can_sel", base + 0x20, 2, 6); + hws[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_hw_divider("uart_serial_podf", "uart_sel", base + 0x24, 0, 6); + hws[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0", 2, 7); + hws[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1", 2, 7); } else { - clk[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_divider("ecspi_root", "pll3_60m", base + 0x38, 19, 6); - clk[IMX6QDL_CLK_CAN_ROOT] = imx_clk_divider("can_root", "pll3_60m", base + 0x20, 2, 6); - clk[IMX6QDL_CLK_IPG_PER] = imx_clk_fixup_divider("ipg_per", "ipg", base + 0x1c, 0, 6, imx_cscmr1_fixup); - clk[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_divider("uart_serial_podf", "pll3_80m", base + 0x24, 0, 6); - clk[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); - clk[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7); + hws[IMX6QDL_CLK_ECSPI_ROOT] = imx_clk_hw_divider("ecspi_root", "pll3_60m", base + 0x38, 19, 6); + hws[IMX6QDL_CLK_CAN_ROOT] = imx_clk_hw_divider("can_root", "pll3_60m", base + 0x20, 2, 6); + hws[IMX6QDL_CLK_IPG_PER] = imx_clk_hw_fixup_divider("ipg_per", "ipg", base + 0x1c, 0, 6, imx_cscmr1_fixup); + hws[IMX6QDL_CLK_UART_SERIAL_PODF] = imx_clk_hw_divider("uart_serial_podf", "pll3_80m", base + 0x24, 0, 6); + hws[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); + hws[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7); } + if (clk_on_imx6dl()) - clk[IMX6QDL_CLK_MLB_PODF] = imx_clk_divider("mlb_podf", "mlb_sel", base + 0x18, 23, 3); + hws[IMX6QDL_CLK_MLB_PODF] = imx_clk_hw_divider("mlb_podf", "mlb_sel", base + 0x18, 23, 3); else - clk[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 23, 3); - clk[IMX6QDL_CLK_GPU3D_CORE_PODF] = imx_clk_divider("gpu3d_core_podf", "gpu3d_core_sel", base + 0x18, 26, 3); + hws[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_hw_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 23, 3); + hws[IMX6QDL_CLK_GPU3D_CORE_PODF] = imx_clk_hw_divider("gpu3d_core_podf", "gpu3d_core_sel", base + 0x18, 26, 3); if (clk_on_imx6dl()) - clk[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 29, 3); + hws[IMX6QDL_CLK_GPU2D_CORE_PODF] = imx_clk_hw_divider("gpu2d_core_podf", "gpu2d_core_sel", base + 0x18, 29, 3); else - clk[IMX6QDL_CLK_GPU3D_SHADER] = imx_clk_divider("gpu3d_shader", "gpu3d_shader_sel", base + 0x18, 29, 3); - clk[IMX6QDL_CLK_IPU1_PODF] = imx_clk_divider("ipu1_podf", "ipu1_sel", base + 0x3c, 11, 3); - clk[IMX6QDL_CLK_IPU2_PODF] = imx_clk_divider("ipu2_podf", "ipu2_sel", base + 0x3c, 16, 3); - clk[IMX6QDL_CLK_LDB_DI0_PODF] = imx_clk_divider_flags("ldb_di0_podf", "ldb_di0_div_3_5", base + 0x20, 10, 1, 0); - clk[IMX6QDL_CLK_LDB_DI1_PODF] = imx_clk_divider_flags("ldb_di1_podf", "ldb_di1_div_3_5", base + 0x20, 11, 1, 0); - clk[IMX6QDL_CLK_IPU1_DI0_PRE] = imx_clk_divider("ipu1_di0_pre", "ipu1_di0_pre_sel", base + 0x34, 3, 3); - clk[IMX6QDL_CLK_IPU1_DI1_PRE] = imx_clk_divider("ipu1_di1_pre", "ipu1_di1_pre_sel", base + 0x34, 12, 3); - clk[IMX6QDL_CLK_IPU2_DI0_PRE] = imx_clk_divider("ipu2_di0_pre", "ipu2_di0_pre_sel", base + 0x38, 3, 3); - clk[IMX6QDL_CLK_IPU2_DI1_PRE] = imx_clk_divider("ipu2_di1_pre", "ipu2_di1_pre_sel", base + 0x38, 12, 3); - clk[IMX6QDL_CLK_HSI_TX_PODF] = imx_clk_divider("hsi_tx_podf", "hsi_tx_sel", base + 0x30, 29, 3); - clk[IMX6QDL_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); - clk[IMX6QDL_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); - clk[IMX6QDL_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); - clk[IMX6QDL_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); - clk[IMX6QDL_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); - clk[IMX6QDL_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); - clk[IMX6QDL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); - clk[IMX6QDL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); - clk[IMX6QDL_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); - clk[IMX6QDL_CLK_USDHC4_PODF] = imx_clk_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3); - clk[IMX6QDL_CLK_ENFC_PRED] = imx_clk_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3); - clk[IMX6QDL_CLK_ENFC_PODF] = imx_clk_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6); + hws[IMX6QDL_CLK_GPU3D_SHADER] = imx_clk_hw_divider("gpu3d_shader", "gpu3d_shader_sel", base + 0x18, 29, 3); + hws[IMX6QDL_CLK_IPU1_PODF] = imx_clk_hw_divider("ipu1_podf", "ipu1_sel", base + 0x3c, 11, 3); + hws[IMX6QDL_CLK_IPU2_PODF] = imx_clk_hw_divider("ipu2_podf", "ipu2_sel", base + 0x3c, 16, 3); + hws[IMX6QDL_CLK_LDB_DI0_PODF] = imx_clk_hw_divider_flags("ldb_di0_podf", "ldb_di0_div_3_5", base + 0x20, 10, 1, 0); + hws[IMX6QDL_CLK_LDB_DI1_PODF] = imx_clk_hw_divider_flags("ldb_di1_podf", "ldb_di1_div_3_5", base + 0x20, 11, 1, 0); + hws[IMX6QDL_CLK_IPU1_DI0_PRE] = imx_clk_hw_divider("ipu1_di0_pre", "ipu1_di0_pre_sel", base + 0x34, 3, 3); + hws[IMX6QDL_CLK_IPU1_DI1_PRE] = imx_clk_hw_divider("ipu1_di1_pre", "ipu1_di1_pre_sel", base + 0x34, 12, 3); + hws[IMX6QDL_CLK_IPU2_DI0_PRE] = imx_clk_hw_divider("ipu2_di0_pre", "ipu2_di0_pre_sel", base + 0x38, 3, 3); + hws[IMX6QDL_CLK_IPU2_DI1_PRE] = imx_clk_hw_divider("ipu2_di1_pre", "ipu2_di1_pre_sel", base + 0x38, 12, 3); + hws[IMX6QDL_CLK_HSI_TX_PODF] = imx_clk_hw_divider("hsi_tx_podf", "hsi_tx_sel", base + 0x30, 29, 3); + hws[IMX6QDL_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); + hws[IMX6QDL_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); + hws[IMX6QDL_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); + hws[IMX6QDL_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); + hws[IMX6QDL_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); + hws[IMX6QDL_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); + hws[IMX6QDL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); + hws[IMX6QDL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); + hws[IMX6QDL_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); + hws[IMX6QDL_CLK_USDHC4_PODF] = imx_clk_hw_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3); + hws[IMX6QDL_CLK_ENFC_PRED] = imx_clk_hw_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3); + hws[IMX6QDL_CLK_ENFC_PODF] = imx_clk_hw_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6); if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_EIM_PODF] = imx_clk_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3); - clk[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3); + hws[IMX6QDL_CLK_EIM_PODF] = imx_clk_hw_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3); + hws[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_hw_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3); } else { - clk[IMX6QDL_CLK_EIM_PODF] = imx_clk_fixup_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3, imx_cscmr1_fixup); - clk[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_fixup_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3, imx_cscmr1_fixup); + hws[IMX6QDL_CLK_EIM_PODF] = imx_clk_hw_fixup_divider("eim_podf", "eim_sel", base + 0x1c, 20, 3, imx_cscmr1_fixup); + hws[IMX6QDL_CLK_EIM_SLOW_PODF] = imx_clk_hw_fixup_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3, imx_cscmr1_fixup); } - clk[IMX6QDL_CLK_VPU_AXI_PODF] = imx_clk_divider("vpu_axi_podf", "vpu_axi_sel", base + 0x24, 25, 3); - clk[IMX6QDL_CLK_CKO1_PODF] = imx_clk_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3); - clk[IMX6QDL_CLK_CKO2_PODF] = imx_clk_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3); + + hws[IMX6QDL_CLK_VPU_AXI_PODF] = imx_clk_hw_divider("vpu_axi_podf", "vpu_axi_sel", base + 0x24, 25, 3); + hws[IMX6QDL_CLK_CKO1_PODF] = imx_clk_hw_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3); + hws[IMX6QDL_CLK_CKO2_PODF] = imx_clk_hw_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3); /* name parent_name reg shift width busy: reg, shift */ - clk[IMX6QDL_CLK_AXI] = imx_clk_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0); - clk[IMX6QDL_CLK_MMDC_CH0_AXI_PODF] = imx_clk_busy_divider("mmdc_ch0_axi_podf", "periph", base + 0x14, 19, 3, base + 0x48, 4); + hws[IMX6QDL_CLK_AXI] = imx_clk_hw_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0); + hws[IMX6QDL_CLK_MMDC_CH0_AXI_PODF] = imx_clk_hw_busy_divider("mmdc_ch0_axi_podf", "periph", base + 0x14, 19, 3, base + 0x48, 4); if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_MMDC_CH1_AXI_CG] = imx_clk_gate("mmdc_ch1_axi_cg", "periph2", base + 0x4, 18); - clk[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_busy_divider("mmdc_ch1_axi_podf", "mmdc_ch1_axi_cg", base + 0x14, 3, 3, base + 0x48, 2); + hws[IMX6QDL_CLK_MMDC_CH1_AXI_CG] = imx_clk_hw_gate("mmdc_ch1_axi_cg", "periph2", base + 0x4, 18); + hws[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_hw_busy_divider("mmdc_ch1_axi_podf", "mmdc_ch1_axi_cg", base + 0x14, 3, 3, base + 0x48, 2); } else { - clk[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_busy_divider("mmdc_ch1_axi_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); + hws[IMX6QDL_CLK_MMDC_CH1_AXI_PODF] = imx_clk_hw_busy_divider("mmdc_ch1_axi_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); } - clk[IMX6QDL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); - clk[IMX6QDL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); + hws[IMX6QDL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); + hws[IMX6QDL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); /* name parent_name reg shift */ - clk[IMX6QDL_CLK_APBH_DMA] = imx_clk_gate2("apbh_dma", "usdhc3", base + 0x68, 4); - clk[IMX6QDL_CLK_ASRC] = imx_clk_gate2_shared("asrc", "asrc_podf", base + 0x68, 6, &share_count_asrc); - clk[IMX6QDL_CLK_ASRC_IPG] = imx_clk_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc); - clk[IMX6QDL_CLK_ASRC_MEM] = imx_clk_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc); - clk[IMX6QDL_CLK_CAAM_MEM] = imx_clk_gate2("caam_mem", "ahb", base + 0x68, 8); - clk[IMX6QDL_CLK_CAAM_ACLK] = imx_clk_gate2("caam_aclk", "ahb", base + 0x68, 10); - clk[IMX6QDL_CLK_CAAM_IPG] = imx_clk_gate2("caam_ipg", "ipg", base + 0x68, 12); - clk[IMX6QDL_CLK_CAN1_IPG] = imx_clk_gate2("can1_ipg", "ipg", base + 0x68, 14); - clk[IMX6QDL_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_root", base + 0x68, 16); - clk[IMX6QDL_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18); - clk[IMX6QDL_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_root", base + 0x68, 20); - clk[IMX6QDL_CLK_DCIC1] = imx_clk_gate2("dcic1", "ipu1_podf", base + 0x68, 24); - clk[IMX6QDL_CLK_DCIC2] = imx_clk_gate2("dcic2", "ipu2_podf", base + 0x68, 26); - clk[IMX6QDL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_root", base + 0x6c, 0); - clk[IMX6QDL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_root", base + 0x6c, 2); - clk[IMX6QDL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_root", base + 0x6c, 4); - clk[IMX6QDL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_root", base + 0x6c, 6); + hws[IMX6QDL_CLK_APBH_DMA] = imx_clk_hw_gate2("apbh_dma", "usdhc3", base + 0x68, 4); + hws[IMX6QDL_CLK_ASRC] = imx_clk_hw_gate2_shared("asrc", "asrc_podf", base + 0x68, 6, &share_count_asrc); + hws[IMX6QDL_CLK_ASRC_IPG] = imx_clk_hw_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc); + hws[IMX6QDL_CLK_ASRC_MEM] = imx_clk_hw_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc); + hws[IMX6QDL_CLK_CAAM_MEM] = imx_clk_hw_gate2("caam_mem", "ahb", base + 0x68, 8); + hws[IMX6QDL_CLK_CAAM_ACLK] = imx_clk_hw_gate2("caam_aclk", "ahb", base + 0x68, 10); + hws[IMX6QDL_CLK_CAAM_IPG] = imx_clk_hw_gate2("caam_ipg", "ipg", base + 0x68, 12); + hws[IMX6QDL_CLK_CAN1_IPG] = imx_clk_hw_gate2("can1_ipg", "ipg", base + 0x68, 14); + hws[IMX6QDL_CLK_CAN1_SERIAL] = imx_clk_hw_gate2("can1_serial", "can_root", base + 0x68, 16); + hws[IMX6QDL_CLK_CAN2_IPG] = imx_clk_hw_gate2("can2_ipg", "ipg", base + 0x68, 18); + hws[IMX6QDL_CLK_CAN2_SERIAL] = imx_clk_hw_gate2("can2_serial", "can_root", base + 0x68, 20); + hws[IMX6QDL_CLK_DCIC1] = imx_clk_hw_gate2("dcic1", "ipu1_podf", base + 0x68, 24); + hws[IMX6QDL_CLK_DCIC2] = imx_clk_hw_gate2("dcic2", "ipu2_podf", base + 0x68, 26); + hws[IMX6QDL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_root", base + 0x6c, 0); + hws[IMX6QDL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_root", base + 0x6c, 2); + hws[IMX6QDL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_root", base + 0x6c, 4); + hws[IMX6QDL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_root", base + 0x6c, 6); if (clk_on_imx6dl()) - clk[IMX6DL_CLK_I2C4] = imx_clk_gate2("i2c4", "ipg_per", base + 0x6c, 8); + hws[IMX6DL_CLK_I2C4] = imx_clk_hw_gate2("i2c4", "ipg_per", base + 0x6c, 8); else - clk[IMX6Q_CLK_ECSPI5] = imx_clk_gate2("ecspi5", "ecspi_root", base + 0x6c, 8); - clk[IMX6QDL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x6c, 10); - clk[IMX6QDL_CLK_EPIT1] = imx_clk_gate2("epit1", "ipg", base + 0x6c, 12); - clk[IMX6QDL_CLK_EPIT2] = imx_clk_gate2("epit2", "ipg", base + 0x6c, 14); - clk[IMX6QDL_CLK_ESAI_EXTAL] = imx_clk_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai); - clk[IMX6QDL_CLK_ESAI_IPG] = imx_clk_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai); - clk[IMX6QDL_CLK_ESAI_MEM] = imx_clk_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai); - clk[IMX6QDL_CLK_GPT_IPG] = imx_clk_gate2("gpt_ipg", "ipg", base + 0x6c, 20); - clk[IMX6QDL_CLK_GPT_IPG_PER] = imx_clk_gate2("gpt_ipg_per", "ipg_per", base + 0x6c, 22); - clk[IMX6QDL_CLK_GPU2D_CORE] = imx_clk_gate2("gpu2d_core", "gpu2d_core_podf", base + 0x6c, 24); - clk[IMX6QDL_CLK_GPU3D_CORE] = imx_clk_gate2("gpu3d_core", "gpu3d_core_podf", base + 0x6c, 26); - clk[IMX6QDL_CLK_HDMI_IAHB] = imx_clk_gate2("hdmi_iahb", "ahb", base + 0x70, 0); - clk[IMX6QDL_CLK_HDMI_ISFR] = imx_clk_gate2("hdmi_isfr", "mipi_core_cfg", base + 0x70, 4); - clk[IMX6QDL_CLK_I2C1] = imx_clk_gate2("i2c1", "ipg_per", base + 0x70, 6); - clk[IMX6QDL_CLK_I2C2] = imx_clk_gate2("i2c2", "ipg_per", base + 0x70, 8); - clk[IMX6QDL_CLK_I2C3] = imx_clk_gate2("i2c3", "ipg_per", base + 0x70, 10); - clk[IMX6QDL_CLK_IIM] = imx_clk_gate2("iim", "ipg", base + 0x70, 12); - clk[IMX6QDL_CLK_ENFC] = imx_clk_gate2("enfc", "enfc_podf", base + 0x70, 14); - clk[IMX6QDL_CLK_VDOA] = imx_clk_gate2("vdoa", "vdo_axi", base + 0x70, 26); - clk[IMX6QDL_CLK_IPU1] = imx_clk_gate2("ipu1", "ipu1_podf", base + 0x74, 0); - clk[IMX6QDL_CLK_IPU1_DI0] = imx_clk_gate2("ipu1_di0", "ipu1_di0_sel", base + 0x74, 2); - clk[IMX6QDL_CLK_IPU1_DI1] = imx_clk_gate2("ipu1_di1", "ipu1_di1_sel", base + 0x74, 4); - clk[IMX6QDL_CLK_IPU2] = imx_clk_gate2("ipu2", "ipu2_podf", base + 0x74, 6); - clk[IMX6QDL_CLK_IPU2_DI0] = imx_clk_gate2("ipu2_di0", "ipu2_di0_sel", base + 0x74, 8); + hws[IMX6Q_CLK_ECSPI5] = imx_clk_hw_gate2("ecspi5", "ecspi_root", base + 0x6c, 8); + hws[IMX6QDL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x6c, 10); + hws[IMX6QDL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "ipg", base + 0x6c, 12); + hws[IMX6QDL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "ipg", base + 0x6c, 14); + hws[IMX6QDL_CLK_ESAI_EXTAL] = imx_clk_hw_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai); + hws[IMX6QDL_CLK_ESAI_IPG] = imx_clk_hw_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai); + hws[IMX6QDL_CLK_ESAI_MEM] = imx_clk_hw_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai); + hws[IMX6QDL_CLK_GPT_IPG] = imx_clk_hw_gate2("gpt_ipg", "ipg", base + 0x6c, 20); + hws[IMX6QDL_CLK_GPT_IPG_PER] = imx_clk_hw_gate2("gpt_ipg_per", "ipg_per", base + 0x6c, 22); + hws[IMX6QDL_CLK_GPU2D_CORE] = imx_clk_hw_gate2("gpu2d_core", "gpu2d_core_podf", base + 0x6c, 24); + hws[IMX6QDL_CLK_GPU3D_CORE] = imx_clk_hw_gate2("gpu3d_core", "gpu3d_core_podf", base + 0x6c, 26); + hws[IMX6QDL_CLK_HDMI_IAHB] = imx_clk_hw_gate2("hdmi_iahb", "ahb", base + 0x70, 0); + hws[IMX6QDL_CLK_HDMI_ISFR] = imx_clk_hw_gate2("hdmi_isfr", "mipi_core_cfg", base + 0x70, 4); + hws[IMX6QDL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "ipg_per", base + 0x70, 6); + hws[IMX6QDL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "ipg_per", base + 0x70, 8); + hws[IMX6QDL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "ipg_per", base + 0x70, 10); + hws[IMX6QDL_CLK_IIM] = imx_clk_hw_gate2("iim", "ipg", base + 0x70, 12); + hws[IMX6QDL_CLK_ENFC] = imx_clk_hw_gate2("enfc", "enfc_podf", base + 0x70, 14); + hws[IMX6QDL_CLK_VDOA] = imx_clk_hw_gate2("vdoa", "vdo_axi", base + 0x70, 26); + hws[IMX6QDL_CLK_IPU1] = imx_clk_hw_gate2("ipu1", "ipu1_podf", base + 0x74, 0); + hws[IMX6QDL_CLK_IPU1_DI0] = imx_clk_hw_gate2("ipu1_di0", "ipu1_di0_sel", base + 0x74, 2); + hws[IMX6QDL_CLK_IPU1_DI1] = imx_clk_hw_gate2("ipu1_di1", "ipu1_di1_sel", base + 0x74, 4); + hws[IMX6QDL_CLK_IPU2] = imx_clk_hw_gate2("ipu2", "ipu2_podf", base + 0x74, 6); + hws[IMX6QDL_CLK_IPU2_DI0] = imx_clk_hw_gate2("ipu2_di0", "ipu2_di0_sel", base + 0x74, 8); if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_sel", base + 0x74, 12); - clk[IMX6QDL_CLK_LDB_DI1] = imx_clk_gate2("ldb_di1", "ldb_di1_sel", base + 0x74, 14); + hws[IMX6QDL_CLK_LDB_DI0] = imx_clk_hw_gate2("ldb_di0", "ldb_di0_sel", base + 0x74, 12); + hws[IMX6QDL_CLK_LDB_DI1] = imx_clk_hw_gate2("ldb_di1", "ldb_di1_sel", base + 0x74, 14); } else { - clk[IMX6QDL_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_podf", base + 0x74, 12); - clk[IMX6QDL_CLK_LDB_DI1] = imx_clk_gate2("ldb_di1", "ldb_di1_podf", base + 0x74, 14); + hws[IMX6QDL_CLK_LDB_DI0] = imx_clk_hw_gate2("ldb_di0", "ldb_di0_podf", base + 0x74, 12); + hws[IMX6QDL_CLK_LDB_DI1] = imx_clk_hw_gate2("ldb_di1", "ldb_di1_podf", base + 0x74, 14); } - clk[IMX6QDL_CLK_IPU2_DI1] = imx_clk_gate2("ipu2_di1", "ipu2_di1_sel", base + 0x74, 10); - clk[IMX6QDL_CLK_HSI_TX] = imx_clk_gate2_shared("hsi_tx", "hsi_tx_podf", base + 0x74, 16, &share_count_mipi_core_cfg); - clk[IMX6QDL_CLK_MIPI_CORE_CFG] = imx_clk_gate2_shared("mipi_core_cfg", "video_27m", base + 0x74, 16, &share_count_mipi_core_cfg); - clk[IMX6QDL_CLK_MIPI_IPG] = imx_clk_gate2_shared("mipi_ipg", "ipg", base + 0x74, 16, &share_count_mipi_core_cfg); + hws[IMX6QDL_CLK_IPU2_DI1] = imx_clk_hw_gate2("ipu2_di1", "ipu2_di1_sel", base + 0x74, 10); + hws[IMX6QDL_CLK_HSI_TX] = imx_clk_hw_gate2_shared("hsi_tx", "hsi_tx_podf", base + 0x74, 16, &share_count_mipi_core_cfg); + hws[IMX6QDL_CLK_MIPI_CORE_CFG] = imx_clk_hw_gate2_shared("mipi_core_cfg", "video_27m", base + 0x74, 16, &share_count_mipi_core_cfg); + hws[IMX6QDL_CLK_MIPI_IPG] = imx_clk_hw_gate2_shared("mipi_ipg", "ipg", base + 0x74, 16, &share_count_mipi_core_cfg); + if (clk_on_imx6dl()) /* * The multiplexer and divider of the imx6q clock gpu2d get * redefined/reused as mlb_sys_sel and mlb_sys_clk_podf on imx6dl. */ - clk[IMX6QDL_CLK_MLB] = imx_clk_gate2("mlb", "mlb_podf", base + 0x74, 18); + hws[IMX6QDL_CLK_MLB] = imx_clk_hw_gate2("mlb", "mlb_podf", base + 0x74, 18); else - clk[IMX6QDL_CLK_MLB] = imx_clk_gate2("mlb", "axi", base + 0x74, 18); - clk[IMX6QDL_CLK_MMDC_CH0_AXI] = imx_clk_gate2_flags("mmdc_ch0_axi", "mmdc_ch0_axi_podf", base + 0x74, 20, CLK_IS_CRITICAL); - clk[IMX6QDL_CLK_MMDC_CH1_AXI] = imx_clk_gate2("mmdc_ch1_axi", "mmdc_ch1_axi_podf", base + 0x74, 22); - clk[IMX6QDL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); - clk[IMX6QDL_CLK_OCRAM] = imx_clk_gate2("ocram", "ahb", base + 0x74, 28); - clk[IMX6QDL_CLK_OPENVG_AXI] = imx_clk_gate2("openvg_axi", "axi", base + 0x74, 30); - clk[IMX6QDL_CLK_PCIE_AXI] = imx_clk_gate2("pcie_axi", "pcie_axi_sel", base + 0x78, 0); - clk[IMX6QDL_CLK_PER1_BCH] = imx_clk_gate2("per1_bch", "usdhc3", base + 0x78, 12); - clk[IMX6QDL_CLK_PWM1] = imx_clk_gate2("pwm1", "ipg_per", base + 0x78, 16); - clk[IMX6QDL_CLK_PWM2] = imx_clk_gate2("pwm2", "ipg_per", base + 0x78, 18); - clk[IMX6QDL_CLK_PWM3] = imx_clk_gate2("pwm3", "ipg_per", base + 0x78, 20); - clk[IMX6QDL_CLK_PWM4] = imx_clk_gate2("pwm4", "ipg_per", base + 0x78, 22); - clk[IMX6QDL_CLK_GPMI_BCH_APB] = imx_clk_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24); - clk[IMX6QDL_CLK_GPMI_BCH] = imx_clk_gate2("gpmi_bch", "usdhc4", base + 0x78, 26); - clk[IMX6QDL_CLK_GPMI_IO] = imx_clk_gate2("gpmi_io", "enfc", base + 0x78, 28); - clk[IMX6QDL_CLK_GPMI_APB] = imx_clk_gate2("gpmi_apb", "usdhc3", base + 0x78, 30); - clk[IMX6QDL_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); - clk[IMX6QDL_CLK_SATA] = imx_clk_gate2("sata", "ahb", base + 0x7c, 4); - clk[IMX6QDL_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6); - clk[IMX6QDL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12); - clk[IMX6QDL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_spdif); - clk[IMX6QDL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif); - clk[IMX6QDL_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); - clk[IMX6QDL_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); - clk[IMX6QDL_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); - clk[IMX6QDL_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); - clk[IMX6QDL_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); - clk[IMX6QDL_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); - clk[IMX6QDL_CLK_UART_IPG] = imx_clk_gate2("uart_ipg", "ipg", base + 0x7c, 24); - clk[IMX6QDL_CLK_UART_SERIAL] = imx_clk_gate2("uart_serial", "uart_serial_podf", base + 0x7c, 26); - clk[IMX6QDL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0); - clk[IMX6QDL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); - clk[IMX6QDL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); - clk[IMX6QDL_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); - clk[IMX6QDL_CLK_USDHC4] = imx_clk_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8); - clk[IMX6QDL_CLK_EIM_SLOW] = imx_clk_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10); - clk[IMX6QDL_CLK_VDO_AXI] = imx_clk_gate2("vdo_axi", "vdo_axi_sel", base + 0x80, 12); - clk[IMX6QDL_CLK_VPU_AXI] = imx_clk_gate2("vpu_axi", "vpu_axi_podf", base + 0x80, 14); + hws[IMX6QDL_CLK_MLB] = imx_clk_hw_gate2("mlb", "axi", base + 0x74, 18); + hws[IMX6QDL_CLK_MMDC_CH0_AXI] = imx_clk_hw_gate2_flags("mmdc_ch0_axi", "mmdc_ch0_axi_podf", base + 0x74, 20, CLK_IS_CRITICAL); + hws[IMX6QDL_CLK_MMDC_CH1_AXI] = imx_clk_hw_gate2("mmdc_ch1_axi", "mmdc_ch1_axi_podf", base + 0x74, 22); + hws[IMX6QDL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); + hws[IMX6QDL_CLK_OCRAM] = imx_clk_hw_gate2("ocram", "ahb", base + 0x74, 28); + hws[IMX6QDL_CLK_OPENVG_AXI] = imx_clk_hw_gate2("openvg_axi", "axi", base + 0x74, 30); + hws[IMX6QDL_CLK_PCIE_AXI] = imx_clk_hw_gate2("pcie_axi", "pcie_axi_sel", base + 0x78, 0); + hws[IMX6QDL_CLK_PER1_BCH] = imx_clk_hw_gate2("per1_bch", "usdhc3", base + 0x78, 12); + hws[IMX6QDL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "ipg_per", base + 0x78, 16); + hws[IMX6QDL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "ipg_per", base + 0x78, 18); + hws[IMX6QDL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "ipg_per", base + 0x78, 20); + hws[IMX6QDL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "ipg_per", base + 0x78, 22); + hws[IMX6QDL_CLK_GPMI_BCH_APB] = imx_clk_hw_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24); + hws[IMX6QDL_CLK_GPMI_BCH] = imx_clk_hw_gate2("gpmi_bch", "usdhc4", base + 0x78, 26); + hws[IMX6QDL_CLK_GPMI_IO] = imx_clk_hw_gate2("gpmi_io", "enfc", base + 0x78, 28); + hws[IMX6QDL_CLK_GPMI_APB] = imx_clk_hw_gate2("gpmi_apb", "usdhc3", base + 0x78, 30); + hws[IMX6QDL_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); + hws[IMX6QDL_CLK_SATA] = imx_clk_hw_gate2("sata", "ahb", base + 0x7c, 4); + hws[IMX6QDL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6); + hws[IMX6QDL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12); + hws[IMX6QDL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_spdif); + hws[IMX6QDL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif); + hws[IMX6QDL_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6QDL_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6QDL_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6QDL_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6QDL_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6QDL_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6QDL_CLK_UART_IPG] = imx_clk_hw_gate2("uart_ipg", "ipg", base + 0x7c, 24); + hws[IMX6QDL_CLK_UART_SERIAL] = imx_clk_hw_gate2("uart_serial", "uart_serial_podf", base + 0x7c, 26); + hws[IMX6QDL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0); + hws[IMX6QDL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); + hws[IMX6QDL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); + hws[IMX6QDL_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); + hws[IMX6QDL_CLK_USDHC4] = imx_clk_hw_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8); + hws[IMX6QDL_CLK_EIM_SLOW] = imx_clk_hw_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10); + hws[IMX6QDL_CLK_VDO_AXI] = imx_clk_hw_gate2("vdo_axi", "vdo_axi_sel", base + 0x80, 12); + hws[IMX6QDL_CLK_VPU_AXI] = imx_clk_hw_gate2("vpu_axi", "vpu_axi_podf", base + 0x80, 14); if (clk_on_imx6qp()) { - clk[IMX6QDL_CLK_PRE0] = imx_clk_gate2("pre0", "pre_axi", base + 0x80, 16); - clk[IMX6QDL_CLK_PRE1] = imx_clk_gate2("pre1", "pre_axi", base + 0x80, 18); - clk[IMX6QDL_CLK_PRE2] = imx_clk_gate2("pre2", "pre_axi", base + 0x80, 20); - clk[IMX6QDL_CLK_PRE3] = imx_clk_gate2("pre3", "pre_axi", base + 0x80, 22); - clk[IMX6QDL_CLK_PRG0_AXI] = imx_clk_gate2_shared("prg0_axi", "ipu1_podf", base + 0x80, 24, &share_count_prg0); - clk[IMX6QDL_CLK_PRG1_AXI] = imx_clk_gate2_shared("prg1_axi", "ipu2_podf", base + 0x80, 26, &share_count_prg1); - clk[IMX6QDL_CLK_PRG0_APB] = imx_clk_gate2_shared("prg0_apb", "ipg", base + 0x80, 24, &share_count_prg0); - clk[IMX6QDL_CLK_PRG1_APB] = imx_clk_gate2_shared("prg1_apb", "ipg", base + 0x80, 26, &share_count_prg1); + hws[IMX6QDL_CLK_PRE0] = imx_clk_hw_gate2("pre0", "pre_axi", base + 0x80, 16); + hws[IMX6QDL_CLK_PRE1] = imx_clk_hw_gate2("pre1", "pre_axi", base + 0x80, 18); + hws[IMX6QDL_CLK_PRE2] = imx_clk_hw_gate2("pre2", "pre_axi", base + 0x80, 20); + hws[IMX6QDL_CLK_PRE3] = imx_clk_hw_gate2("pre3", "pre_axi", base + 0x80, 22); + hws[IMX6QDL_CLK_PRG0_AXI] = imx_clk_hw_gate2_shared("prg0_axi", "ipu1_podf", base + 0x80, 24, &share_count_prg0); + hws[IMX6QDL_CLK_PRG1_AXI] = imx_clk_hw_gate2_shared("prg1_axi", "ipu2_podf", base + 0x80, 26, &share_count_prg1); + hws[IMX6QDL_CLK_PRG0_APB] = imx_clk_hw_gate2_shared("prg0_apb", "ipg", base + 0x80, 24, &share_count_prg0); + hws[IMX6QDL_CLK_PRG1_APB] = imx_clk_hw_gate2_shared("prg1_apb", "ipg", base + 0x80, 26, &share_count_prg1); } - clk[IMX6QDL_CLK_CKO1] = imx_clk_gate("cko1", "cko1_podf", base + 0x60, 7); - clk[IMX6QDL_CLK_CKO2] = imx_clk_gate("cko2", "cko2_podf", base + 0x60, 24); + hws[IMX6QDL_CLK_CKO1] = imx_clk_hw_gate("cko1", "cko1_podf", base + 0x60, 7); + hws[IMX6QDL_CLK_CKO2] = imx_clk_hw_gate("cko2", "cko2_podf", base + 0x60, 24); /* * The gpt_3m clock is not available on i.MX6Q TO1.0. Let's point it * to clock gpt_ipg_per to ease the gpt driver code. */ if (clk_on_imx6q() && imx_get_soc_revision() == IMX_CHIP_REVISION_1_0) - clk[IMX6QDL_CLK_GPT_3M] = clk[IMX6QDL_CLK_GPT_IPG_PER]; + hws[IMX6QDL_CLK_GPT_3M] = hws[IMX6QDL_CLK_GPT_IPG_PER]; - imx_check_clocks(clk, ARRAY_SIZE(clk)); + imx_check_clk_hws(hws, IMX6QDL_CLK_END); - clk_data.clks = clk; - clk_data.clk_num = ARRAY_SIZE(clk); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); - clk_register_clkdev(clk[IMX6QDL_CLK_ENET_REF], "enet_ref", NULL); + clk_hw_register_clkdev(hws[IMX6QDL_CLK_ENET_REF], "enet_ref", NULL); - clk_set_rate(clk[IMX6QDL_CLK_PLL3_PFD1_540M], 540000000); + clk_set_rate(hws[IMX6QDL_CLK_PLL3_PFD1_540M]->clk, 540000000); if (clk_on_imx6dl()) - clk_set_parent(clk[IMX6QDL_CLK_IPU1_SEL], clk[IMX6QDL_CLK_PLL3_PFD1_540M]); + clk_set_parent(hws[IMX6QDL_CLK_IPU1_SEL]->clk, hws[IMX6QDL_CLK_PLL3_PFD1_540M]->clk); - clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); - clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); - clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); - clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_PRE_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]); - clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI0_SEL], clk[IMX6QDL_CLK_IPU1_DI0_PRE]); - clk_set_parent(clk[IMX6QDL_CLK_IPU1_DI1_SEL], clk[IMX6QDL_CLK_IPU1_DI1_PRE]); - clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI0_SEL], clk[IMX6QDL_CLK_IPU2_DI0_PRE]); - clk_set_parent(clk[IMX6QDL_CLK_IPU2_DI1_SEL], clk[IMX6QDL_CLK_IPU2_DI1_PRE]); + clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI0_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk); + clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI1_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk); + clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI0_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk); + clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI1_PRE_SEL]->clk, hws[IMX6QDL_CLK_PLL5_VIDEO_DIV]->clk); + clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI0_SEL]->clk, hws[IMX6QDL_CLK_IPU1_DI0_PRE]->clk); + clk_set_parent(hws[IMX6QDL_CLK_IPU1_DI1_SEL]->clk, hws[IMX6QDL_CLK_IPU1_DI1_PRE]->clk); + clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI0_SEL]->clk, hws[IMX6QDL_CLK_IPU2_DI0_PRE]->clk); + clk_set_parent(hws[IMX6QDL_CLK_IPU2_DI1_SEL]->clk, hws[IMX6QDL_CLK_IPU2_DI1_PRE]->clk); /* * The gpmi needs 100MHz frequency in the EDO/Sync mode, * We can not get the 100MHz from the pll2_pfd0_352m. * So choose pll2_pfd2_396m as enfc_sel's parent. */ - clk_set_parent(clk[IMX6QDL_CLK_ENFC_SEL], clk[IMX6QDL_CLK_PLL2_PFD2_396M]); + clk_set_parent(hws[IMX6QDL_CLK_ENFC_SEL]->clk, hws[IMX6QDL_CLK_PLL2_PFD2_396M]->clk); if (IS_ENABLED(CONFIG_USB_MXS_PHY)) { - clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY1_GATE]); - clk_prepare_enable(clk[IMX6QDL_CLK_USBPHY2_GATE]); + clk_prepare_enable(hws[IMX6QDL_CLK_USBPHY1_GATE]->clk); + clk_prepare_enable(hws[IMX6QDL_CLK_USBPHY2_GATE]->clk); } /* * Let's initially set up CLKO with OSC24M, since this configuration * is widely used by imx6q board designs to clock audio codec. */ - ret = clk_set_parent(clk[IMX6QDL_CLK_CKO2_SEL], clk[IMX6QDL_CLK_OSC]); + ret = clk_set_parent(hws[IMX6QDL_CLK_CKO2_SEL]->clk, hws[IMX6QDL_CLK_OSC]->clk); if (!ret) - ret = clk_set_parent(clk[IMX6QDL_CLK_CKO], clk[IMX6QDL_CLK_CKO2]); + ret = clk_set_parent(hws[IMX6QDL_CLK_CKO]->clk, hws[IMX6QDL_CLK_CKO2]->clk); if (ret) pr_warn("failed to set up CLKO: %d\n", ret); /* Audio-related clocks configuration */ - clk_set_parent(clk[IMX6QDL_CLK_SPDIF_SEL], clk[IMX6QDL_CLK_PLL3_PFD3_454M]); + clk_set_parent(hws[IMX6QDL_CLK_SPDIF_SEL]->clk, hws[IMX6QDL_CLK_PLL3_PFD3_454M]->clk); /* All existing boards with PCIe use LVDS1 */ if (IS_ENABLED(CONFIG_PCI_IMX6)) - clk_set_parent(clk[IMX6QDL_CLK_LVDS1_SEL], clk[IMX6QDL_CLK_SATA_REF_100M]); + clk_set_parent(hws[IMX6QDL_CLK_LVDS1_SEL]->clk, hws[IMX6QDL_CLK_SATA_REF_100M]->clk); /* * Initialize the GPU clock muxes, so that the maximum specified clock * rates for the respective SoC are not exceeded. */ if (clk_on_imx6dl()) { - clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL], - clk[IMX6QDL_CLK_PLL2_PFD1_594M]); - clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL], - clk[IMX6QDL_CLK_PLL2_PFD1_594M]); + clk_set_parent(hws[IMX6QDL_CLK_GPU3D_CORE_SEL]->clk, + hws[IMX6QDL_CLK_PLL2_PFD1_594M]->clk); + clk_set_parent(hws[IMX6QDL_CLK_GPU2D_CORE_SEL]->clk, + hws[IMX6QDL_CLK_PLL2_PFD1_594M]->clk); } else if (clk_on_imx6q()) { - clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL], - clk[IMX6QDL_CLK_MMDC_CH0_AXI]); - clk_set_parent(clk[IMX6QDL_CLK_GPU3D_SHADER_SEL], - clk[IMX6QDL_CLK_PLL2_PFD1_594M]); - clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL], - clk[IMX6QDL_CLK_PLL3_USB_OTG]); + clk_set_parent(hws[IMX6QDL_CLK_GPU3D_CORE_SEL]->clk, + hws[IMX6QDL_CLK_MMDC_CH0_AXI]->clk); + clk_set_parent(hws[IMX6QDL_CLK_GPU3D_SHADER_SEL]->clk, + hws[IMX6QDL_CLK_PLL2_PFD1_594M]->clk); + clk_set_parent(hws[IMX6QDL_CLK_GPU2D_CORE_SEL]->clk, + hws[IMX6QDL_CLK_PLL3_USB_OTG]->clk); + } + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_clks[i] = &hws[index]->clk; } imx_register_uart_clocks(uart_clks); diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c index f9f1f8a95d92..4bd44d89eaaa 100644 --- a/drivers/clk/imx/clk-imx6sl.c +++ b/drivers/clk/imx/clk-imx6sl.c @@ -13,8 +13,6 @@ #include "clk.h" -#define CCDR 0x4 -#define BM_CCM_CCDR_MMDC_CH0_MASK (1 << 17) #define CCSR 0xc #define BM_CCSR_PLL1_SW_CLK_SEL (1 << 2) #define CACRR 0x10 @@ -97,8 +95,8 @@ static unsigned int share_count_ssi2; static unsigned int share_count_ssi3; static unsigned int share_count_spdif; -static struct clk *clks[IMX6SL_CLK_END]; -static struct clk_onecell_data clk_data; +static struct clk_hw **hws; +static struct clk_hw_onecell_data *clk_hw_data; static void __iomem *ccm_base; static void __iomem *anatop_base; @@ -179,74 +177,84 @@ void imx6sl_set_wait_clk(bool enter) imx6sl_enable_pll_arm(false); } -static struct clk ** const uart_clks[] __initconst = { - &clks[IMX6SL_CLK_UART], - &clks[IMX6SL_CLK_UART_SERIAL], - NULL +static const int uart_clk_ids[] __initconst = { + IMX6SL_CLK_UART, + IMX6SL_CLK_UART_SERIAL, }; +static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; + static void __init imx6sl_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; int ret; + int i; + + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX6SL_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return; + + clk_hw_data->num = IMX6SL_CLK_END; + hws = clk_hw_data->hws; - clks[IMX6SL_CLK_DUMMY] = imx_clk_fixed("dummy", 0); - clks[IMX6SL_CLK_CKIL] = imx_obtain_fixed_clock("ckil", 0); - clks[IMX6SL_CLK_OSC] = imx_obtain_fixed_clock("osc", 0); + hws[IMX6SL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); + hws[IMX6SL_CLK_CKIL] = imx_obtain_fixed_clock_hw("ckil", 0); + hws[IMX6SL_CLK_OSC] = imx_obtain_fixed_clock_hw("osc", 0); /* Clock source from external clock via CLK1 PAD */ - clks[IMX6SL_CLK_ANACLK1] = imx_obtain_fixed_clock("anaclk1", 0); + hws[IMX6SL_CLK_ANACLK1] = imx_obtain_fixed_clock_hw("anaclk1", 0); np = of_find_compatible_node(NULL, NULL, "fsl,imx6sl-anatop"); base = of_iomap(np, 0); WARN_ON(!base); anatop_base = base; - clks[IMX6SL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); /* type name parent_name base div_mask */ - clks[IMX6SL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); - clks[IMX6SL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); - clks[IMX6SL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); - clks[IMX6SL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); - clks[IMX6SL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); - clks[IMX6SL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); - clks[IMX6SL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); - - clks[IMX6SL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + hws[IMX6SL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + hws[IMX6SL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + hws[IMX6SL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + hws[IMX6SL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + hws[IMX6SL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + hws[IMX6SL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); + + hws[IMX6SL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); /* Do not bypass PLLs initially */ - clk_set_parent(clks[IMX6SL_PLL1_BYPASS], clks[IMX6SL_CLK_PLL1]); - clk_set_parent(clks[IMX6SL_PLL2_BYPASS], clks[IMX6SL_CLK_PLL2]); - clk_set_parent(clks[IMX6SL_PLL3_BYPASS], clks[IMX6SL_CLK_PLL3]); - clk_set_parent(clks[IMX6SL_PLL4_BYPASS], clks[IMX6SL_CLK_PLL4]); - clk_set_parent(clks[IMX6SL_PLL5_BYPASS], clks[IMX6SL_CLK_PLL5]); - clk_set_parent(clks[IMX6SL_PLL6_BYPASS], clks[IMX6SL_CLK_PLL6]); - clk_set_parent(clks[IMX6SL_PLL7_BYPASS], clks[IMX6SL_CLK_PLL7]); - - clks[IMX6SL_CLK_PLL1_SYS] = imx_clk_gate("pll1_sys", "pll1_bypass", base + 0x00, 13); - clks[IMX6SL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); - clks[IMX6SL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); - clks[IMX6SL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); - clks[IMX6SL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); - clks[IMX6SL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); - clks[IMX6SL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); - - clks[IMX6SL_CLK_LVDS1_SEL] = imx_clk_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); - clks[IMX6SL_CLK_LVDS1_OUT] = imx_clk_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12)); - clks[IMX6SL_CLK_LVDS1_IN] = imx_clk_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10)); + clk_set_parent(hws[IMX6SL_PLL1_BYPASS]->clk, hws[IMX6SL_CLK_PLL1]->clk); + clk_set_parent(hws[IMX6SL_PLL2_BYPASS]->clk, hws[IMX6SL_CLK_PLL2]->clk); + clk_set_parent(hws[IMX6SL_PLL3_BYPASS]->clk, hws[IMX6SL_CLK_PLL3]->clk); + clk_set_parent(hws[IMX6SL_PLL4_BYPASS]->clk, hws[IMX6SL_CLK_PLL4]->clk); + clk_set_parent(hws[IMX6SL_PLL5_BYPASS]->clk, hws[IMX6SL_CLK_PLL5]->clk); + clk_set_parent(hws[IMX6SL_PLL6_BYPASS]->clk, hws[IMX6SL_CLK_PLL6]->clk); + clk_set_parent(hws[IMX6SL_PLL7_BYPASS]->clk, hws[IMX6SL_CLK_PLL7]->clk); + + hws[IMX6SL_CLK_PLL1_SYS] = imx_clk_hw_gate("pll1_sys", "pll1_bypass", base + 0x00, 13); + hws[IMX6SL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); + hws[IMX6SL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); + hws[IMX6SL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); + hws[IMX6SL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); + hws[IMX6SL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); + hws[IMX6SL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); + + hws[IMX6SL_CLK_LVDS1_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); + hws[IMX6SL_CLK_LVDS1_OUT] = imx_clk_hw_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12)); + hws[IMX6SL_CLK_LVDS1_IN] = imx_clk_hw_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10)); /* * usbphy1 and usbphy2 are implemented as dummy gates using reserve @@ -255,32 +263,32 @@ static void __init imx6sl_clocks_init(struct device_node *ccm_node) * turned on during boot, and software will not need to control it * anymore after that. */ - clks[IMX6SL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); - clks[IMX6SL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); - clks[IMX6SL_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6); - clks[IMX6SL_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6); + hws[IMX6SL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); + hws[IMX6SL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); + hws[IMX6SL_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6); + hws[IMX6SL_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6); /* dev name parent_name flags reg shift width div: flags, div_table lock */ - clks[IMX6SL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6SL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock); - clks[IMX6SL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6SL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); - clks[IMX6SL_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock); + hws[IMX6SL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); + hws[IMX6SL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock); + hws[IMX6SL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); + hws[IMX6SL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); + hws[IMX6SL_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock); /* name parent_name reg idx */ - clks[IMX6SL_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0", "pll2_bus", base + 0x100, 0); - clks[IMX6SL_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1", "pll2_bus", base + 0x100, 1); - clks[IMX6SL_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2", "pll2_bus", base + 0x100, 2); - clks[IMX6SL_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0", "pll3_usb_otg", base + 0xf0, 0); - clks[IMX6SL_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1", "pll3_usb_otg", base + 0xf0, 1); - clks[IMX6SL_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2", "pll3_usb_otg", base + 0xf0, 2); - clks[IMX6SL_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3", "pll3_usb_otg", base + 0xf0, 3); + hws[IMX6SL_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0", "pll2_bus", base + 0x100, 0); + hws[IMX6SL_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1", "pll2_bus", base + 0x100, 1); + hws[IMX6SL_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2", "pll2_bus", base + 0x100, 2); + hws[IMX6SL_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0", "pll3_usb_otg", base + 0xf0, 0); + hws[IMX6SL_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1", "pll3_usb_otg", base + 0xf0, 1); + hws[IMX6SL_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2", "pll3_usb_otg", base + 0xf0, 2); + hws[IMX6SL_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3", "pll3_usb_otg", base + 0xf0, 3); /* name parent_name mult div */ - clks[IMX6SL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2", 1, 2); - clks[IMX6SL_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); - clks[IMX6SL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); - clks[IMX6SL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); + hws[IMX6SL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2", 1, 2); + hws[IMX6SL_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); + hws[IMX6SL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); + hws[IMX6SL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); np = ccm_node; base = of_iomap(np, 0); @@ -288,157 +296,160 @@ static void __init imx6sl_clocks_init(struct device_node *ccm_node) ccm_base = base; /* name reg shift width parent_names num_parents */ - clks[IMX6SL_CLK_STEP] = imx_clk_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels)); - clks[IMX6SL_CLK_PLL1_SW] = imx_clk_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels)); - clks[IMX6SL_CLK_OCRAM_ALT_SEL] = imx_clk_mux("ocram_alt_sel", base + 0x14, 7, 1, ocram_alt_sels, ARRAY_SIZE(ocram_alt_sels)); - clks[IMX6SL_CLK_OCRAM_SEL] = imx_clk_mux("ocram_sel", base + 0x14, 6, 1, ocram_sels, ARRAY_SIZE(ocram_sels)); - clks[IMX6SL_CLK_PRE_PERIPH2_SEL] = imx_clk_mux("pre_periph2_sel", base + 0x18, 21, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels)); - clks[IMX6SL_CLK_PRE_PERIPH_SEL] = imx_clk_mux("pre_periph_sel", base + 0x18, 18, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels)); - clks[IMX6SL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); - clks[IMX6SL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); - clks[IMX6SL_CLK_CSI_SEL] = imx_clk_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels)); - clks[IMX6SL_CLK_LCDIF_AXI_SEL] = imx_clk_mux("lcdif_axi_sel", base + 0x3c, 14, 2, lcdif_axi_sels, ARRAY_SIZE(lcdif_axi_sels)); - clks[IMX6SL_CLK_USDHC1_SEL] = imx_clk_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_USDHC2_SEL] = imx_clk_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_USDHC3_SEL] = imx_clk_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_USDHC4_SEL] = imx_clk_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_SSI1_SEL] = imx_clk_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_SSI2_SEL] = imx_clk_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_SSI3_SEL] = imx_clk_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_PERCLK_SEL] = imx_clk_fixup_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels), imx_cscmr1_fixup); - clks[IMX6SL_CLK_PXP_AXI_SEL] = imx_clk_mux("pxp_axi_sel", base + 0x34, 6, 3, pxp_axi_sels, ARRAY_SIZE(pxp_axi_sels)); - clks[IMX6SL_CLK_EPDC_AXI_SEL] = imx_clk_mux("epdc_axi_sel", base + 0x34, 15, 3, epdc_axi_sels, ARRAY_SIZE(epdc_axi_sels)); - clks[IMX6SL_CLK_GPU2D_OVG_SEL] = imx_clk_mux("gpu2d_ovg_sel", base + 0x18, 4, 2, gpu2d_ovg_sels, ARRAY_SIZE(gpu2d_ovg_sels)); - clks[IMX6SL_CLK_GPU2D_SEL] = imx_clk_mux("gpu2d_sel", base + 0x18, 8, 2, gpu2d_sels, ARRAY_SIZE(gpu2d_sels)); - clks[IMX6SL_CLK_LCDIF_PIX_SEL] = imx_clk_mux("lcdif_pix_sel", base + 0x38, 6, 3, lcdif_pix_sels, ARRAY_SIZE(lcdif_pix_sels)); - clks[IMX6SL_CLK_EPDC_PIX_SEL] = imx_clk_mux("epdc_pix_sel", base + 0x38, 15, 3, epdc_pix_sels, ARRAY_SIZE(epdc_pix_sels)); - clks[IMX6SL_CLK_SPDIF0_SEL] = imx_clk_mux("spdif0_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clks[IMX6SL_CLK_SPDIF1_SEL] = imx_clk_mux("spdif1_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clks[IMX6SL_CLK_EXTERN_AUDIO_SEL] = imx_clk_mux("extern_audio_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clks[IMX6SL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); - clks[IMX6SL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); + hws[IMX6SL_CLK_STEP] = imx_clk_hw_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels)); + hws[IMX6SL_CLK_PLL1_SW] = imx_clk_hw_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels)); + hws[IMX6SL_CLK_OCRAM_ALT_SEL] = imx_clk_hw_mux("ocram_alt_sel", base + 0x14, 7, 1, ocram_alt_sels, ARRAY_SIZE(ocram_alt_sels)); + hws[IMX6SL_CLK_OCRAM_SEL] = imx_clk_hw_mux("ocram_sel", base + 0x14, 6, 1, ocram_sels, ARRAY_SIZE(ocram_sels)); + hws[IMX6SL_CLK_PRE_PERIPH2_SEL] = imx_clk_hw_mux("pre_periph2_sel", base + 0x18, 21, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels)); + hws[IMX6SL_CLK_PRE_PERIPH_SEL] = imx_clk_hw_mux("pre_periph_sel", base + 0x18, 18, 2, pre_periph_sels, ARRAY_SIZE(pre_periph_sels)); + hws[IMX6SL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); + hws[IMX6SL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); + hws[IMX6SL_CLK_CSI_SEL] = imx_clk_hw_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels)); + hws[IMX6SL_CLK_LCDIF_AXI_SEL] = imx_clk_hw_mux("lcdif_axi_sel", base + 0x3c, 14, 2, lcdif_axi_sels, ARRAY_SIZE(lcdif_axi_sels)); + hws[IMX6SL_CLK_USDHC1_SEL] = imx_clk_hw_fixup_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_USDHC2_SEL] = imx_clk_hw_fixup_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_USDHC3_SEL] = imx_clk_hw_fixup_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_USDHC4_SEL] = imx_clk_hw_fixup_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_SSI1_SEL] = imx_clk_hw_fixup_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_SSI2_SEL] = imx_clk_hw_fixup_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_SSI3_SEL] = imx_clk_hw_fixup_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_PERCLK_SEL] = imx_clk_hw_fixup_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels), imx_cscmr1_fixup); + hws[IMX6SL_CLK_PXP_AXI_SEL] = imx_clk_hw_mux("pxp_axi_sel", base + 0x34, 6, 3, pxp_axi_sels, ARRAY_SIZE(pxp_axi_sels)); + hws[IMX6SL_CLK_EPDC_AXI_SEL] = imx_clk_hw_mux("epdc_axi_sel", base + 0x34, 15, 3, epdc_axi_sels, ARRAY_SIZE(epdc_axi_sels)); + hws[IMX6SL_CLK_GPU2D_OVG_SEL] = imx_clk_hw_mux("gpu2d_ovg_sel", base + 0x18, 4, 2, gpu2d_ovg_sels, ARRAY_SIZE(gpu2d_ovg_sels)); + hws[IMX6SL_CLK_GPU2D_SEL] = imx_clk_hw_mux("gpu2d_sel", base + 0x18, 8, 2, gpu2d_sels, ARRAY_SIZE(gpu2d_sels)); + hws[IMX6SL_CLK_LCDIF_PIX_SEL] = imx_clk_hw_mux("lcdif_pix_sel", base + 0x38, 6, 3, lcdif_pix_sels, ARRAY_SIZE(lcdif_pix_sels)); + hws[IMX6SL_CLK_EPDC_PIX_SEL] = imx_clk_hw_mux("epdc_pix_sel", base + 0x38, 15, 3, epdc_pix_sels, ARRAY_SIZE(epdc_pix_sels)); + hws[IMX6SL_CLK_SPDIF0_SEL] = imx_clk_hw_mux("spdif0_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6SL_CLK_SPDIF1_SEL] = imx_clk_hw_mux("spdif1_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6SL_CLK_EXTERN_AUDIO_SEL] = imx_clk_hw_mux("extern_audio_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6SL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); + hws[IMX6SL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); /* name reg shift width busy: reg, shift parent_names num_parents */ - clks[IMX6SL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); - clks[IMX6SL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); + hws[IMX6SL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); + hws[IMX6SL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); /* name parent_name reg shift width */ - clks[IMX6SL_CLK_OCRAM_PODF] = imx_clk_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0); - clks[IMX6SL_CLK_PERIPH_CLK2_PODF] = imx_clk_divider("periph_clk2_podf", "periph_clk2_sel", base + 0x14, 27, 3); - clks[IMX6SL_CLK_PERIPH2_CLK2_PODF] = imx_clk_divider("periph2_clk2_podf", "periph2_clk2_sel", base + 0x14, 0, 3); - clks[IMX6SL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2); - clks[IMX6SL_CLK_CSI_PODF] = imx_clk_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3); - clks[IMX6SL_CLK_LCDIF_AXI_PODF] = imx_clk_divider("lcdif_axi_podf", "lcdif_axi_sel", base + 0x3c, 16, 3); - clks[IMX6SL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); - clks[IMX6SL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); - clks[IMX6SL_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); - clks[IMX6SL_CLK_USDHC4_PODF] = imx_clk_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3); - clks[IMX6SL_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); - clks[IMX6SL_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); - clks[IMX6SL_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); - clks[IMX6SL_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); - clks[IMX6SL_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); - clks[IMX6SL_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); - clks[IMX6SL_CLK_PERCLK] = imx_clk_fixup_divider("perclk", "perclk_sel", base + 0x1c, 0, 6, imx_cscmr1_fixup); - clks[IMX6SL_CLK_PXP_AXI_PODF] = imx_clk_divider("pxp_axi_podf", "pxp_axi_sel", base + 0x34, 3, 3); - clks[IMX6SL_CLK_EPDC_AXI_PODF] = imx_clk_divider("epdc_axi_podf", "epdc_axi_sel", base + 0x34, 12, 3); - clks[IMX6SL_CLK_GPU2D_OVG_PODF] = imx_clk_divider("gpu2d_ovg_podf", "gpu2d_ovg_sel", base + 0x18, 26, 3); - clks[IMX6SL_CLK_GPU2D_PODF] = imx_clk_divider("gpu2d_podf", "gpu2d_sel", base + 0x18, 29, 3); - clks[IMX6SL_CLK_LCDIF_PIX_PRED] = imx_clk_divider("lcdif_pix_pred", "lcdif_pix_sel", base + 0x38, 3, 3); - clks[IMX6SL_CLK_EPDC_PIX_PRED] = imx_clk_divider("epdc_pix_pred", "epdc_pix_sel", base + 0x38, 12, 3); - clks[IMX6SL_CLK_LCDIF_PIX_PODF] = imx_clk_fixup_divider("lcdif_pix_podf", "lcdif_pix_pred", base + 0x1c, 20, 3, imx_cscmr1_fixup); - clks[IMX6SL_CLK_EPDC_PIX_PODF] = imx_clk_divider("epdc_pix_podf", "epdc_pix_pred", base + 0x18, 23, 3); - clks[IMX6SL_CLK_SPDIF0_PRED] = imx_clk_divider("spdif0_pred", "spdif0_sel", base + 0x30, 25, 3); - clks[IMX6SL_CLK_SPDIF0_PODF] = imx_clk_divider("spdif0_podf", "spdif0_pred", base + 0x30, 22, 3); - clks[IMX6SL_CLK_SPDIF1_PRED] = imx_clk_divider("spdif1_pred", "spdif1_sel", base + 0x30, 12, 3); - clks[IMX6SL_CLK_SPDIF1_PODF] = imx_clk_divider("spdif1_podf", "spdif1_pred", base + 0x30, 9, 3); - clks[IMX6SL_CLK_EXTERN_AUDIO_PRED] = imx_clk_divider("extern_audio_pred", "extern_audio_sel", base + 0x28, 9, 3); - clks[IMX6SL_CLK_EXTERN_AUDIO_PODF] = imx_clk_divider("extern_audio_podf", "extern_audio_pred", base + 0x28, 25, 3); - clks[IMX6SL_CLK_ECSPI_ROOT] = imx_clk_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6); - clks[IMX6SL_CLK_UART_ROOT] = imx_clk_divider("uart_root", "uart_sel", base + 0x24, 0, 6); + hws[IMX6SL_CLK_OCRAM_PODF] = imx_clk_hw_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0); + hws[IMX6SL_CLK_PERIPH_CLK2_PODF] = imx_clk_hw_divider("periph_clk2_podf", "periph_clk2_sel", base + 0x14, 27, 3); + hws[IMX6SL_CLK_PERIPH2_CLK2_PODF] = imx_clk_hw_divider("periph2_clk2_podf", "periph2_clk2_sel", base + 0x14, 0, 3); + hws[IMX6SL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2); + hws[IMX6SL_CLK_CSI_PODF] = imx_clk_hw_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3); + hws[IMX6SL_CLK_LCDIF_AXI_PODF] = imx_clk_hw_divider("lcdif_axi_podf", "lcdif_axi_sel", base + 0x3c, 16, 3); + hws[IMX6SL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); + hws[IMX6SL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); + hws[IMX6SL_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); + hws[IMX6SL_CLK_USDHC4_PODF] = imx_clk_hw_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3); + hws[IMX6SL_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); + hws[IMX6SL_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); + hws[IMX6SL_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); + hws[IMX6SL_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); + hws[IMX6SL_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); + hws[IMX6SL_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); + hws[IMX6SL_CLK_PERCLK] = imx_clk_hw_fixup_divider("perclk", "perclk_sel", base + 0x1c, 0, 6, imx_cscmr1_fixup); + hws[IMX6SL_CLK_PXP_AXI_PODF] = imx_clk_hw_divider("pxp_axi_podf", "pxp_axi_sel", base + 0x34, 3, 3); + hws[IMX6SL_CLK_EPDC_AXI_PODF] = imx_clk_hw_divider("epdc_axi_podf", "epdc_axi_sel", base + 0x34, 12, 3); + hws[IMX6SL_CLK_GPU2D_OVG_PODF] = imx_clk_hw_divider("gpu2d_ovg_podf", "gpu2d_ovg_sel", base + 0x18, 26, 3); + hws[IMX6SL_CLK_GPU2D_PODF] = imx_clk_hw_divider("gpu2d_podf", "gpu2d_sel", base + 0x18, 29, 3); + hws[IMX6SL_CLK_LCDIF_PIX_PRED] = imx_clk_hw_divider("lcdif_pix_pred", "lcdif_pix_sel", base + 0x38, 3, 3); + hws[IMX6SL_CLK_EPDC_PIX_PRED] = imx_clk_hw_divider("epdc_pix_pred", "epdc_pix_sel", base + 0x38, 12, 3); + hws[IMX6SL_CLK_LCDIF_PIX_PODF] = imx_clk_hw_fixup_divider("lcdif_pix_podf", "lcdif_pix_pred", base + 0x1c, 20, 3, imx_cscmr1_fixup); + hws[IMX6SL_CLK_EPDC_PIX_PODF] = imx_clk_hw_divider("epdc_pix_podf", "epdc_pix_pred", base + 0x18, 23, 3); + hws[IMX6SL_CLK_SPDIF0_PRED] = imx_clk_hw_divider("spdif0_pred", "spdif0_sel", base + 0x30, 25, 3); + hws[IMX6SL_CLK_SPDIF0_PODF] = imx_clk_hw_divider("spdif0_podf", "spdif0_pred", base + 0x30, 22, 3); + hws[IMX6SL_CLK_SPDIF1_PRED] = imx_clk_hw_divider("spdif1_pred", "spdif1_sel", base + 0x30, 12, 3); + hws[IMX6SL_CLK_SPDIF1_PODF] = imx_clk_hw_divider("spdif1_podf", "spdif1_pred", base + 0x30, 9, 3); + hws[IMX6SL_CLK_EXTERN_AUDIO_PRED] = imx_clk_hw_divider("extern_audio_pred", "extern_audio_sel", base + 0x28, 9, 3); + hws[IMX6SL_CLK_EXTERN_AUDIO_PODF] = imx_clk_hw_divider("extern_audio_podf", "extern_audio_pred", base + 0x28, 25, 3); + hws[IMX6SL_CLK_ECSPI_ROOT] = imx_clk_hw_divider("ecspi_root", "ecspi_sel", base + 0x38, 19, 6); + hws[IMX6SL_CLK_UART_ROOT] = imx_clk_hw_divider("uart_root", "uart_sel", base + 0x24, 0, 6); /* name parent_name reg shift width busy: reg, shift */ - clks[IMX6SL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); - clks[IMX6SL_CLK_MMDC_ROOT] = imx_clk_busy_divider("mmdc", "periph2", base + 0x14, 3, 3, base + 0x48, 2); - clks[IMX6SL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); + hws[IMX6SL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); + hws[IMX6SL_CLK_MMDC_ROOT] = imx_clk_hw_busy_divider("mmdc", "periph2", base + 0x14, 3, 3, base + 0x48, 2); + hws[IMX6SL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); /* name parent_name reg shift */ - clks[IMX6SL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_root", base + 0x6c, 0); - clks[IMX6SL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_root", base + 0x6c, 2); - clks[IMX6SL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_root", base + 0x6c, 4); - clks[IMX6SL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_root", base + 0x6c, 6); - clks[IMX6SL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x6c, 10); - clks[IMX6SL_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12); - clks[IMX6SL_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14); - clks[IMX6SL_CLK_EXTERN_AUDIO] = imx_clk_gate2("extern_audio", "extern_audio_podf", base + 0x6c, 16); - clks[IMX6SL_CLK_GPT] = imx_clk_gate2("gpt", "perclk", base + 0x6c, 20); - clks[IMX6SL_CLK_GPT_SERIAL] = imx_clk_gate2("gpt_serial", "perclk", base + 0x6c, 22); - clks[IMX6SL_CLK_GPU2D_OVG] = imx_clk_gate2("gpu2d_ovg", "gpu2d_ovg_podf", base + 0x6c, 26); - clks[IMX6SL_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6); - clks[IMX6SL_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8); - clks[IMX6SL_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10); - clks[IMX6SL_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12); - clks[IMX6SL_CLK_CSI] = imx_clk_gate2("csi", "csi_podf", base + 0x74, 0); - clks[IMX6SL_CLK_PXP_AXI] = imx_clk_gate2("pxp_axi", "pxp_axi_podf", base + 0x74, 2); - clks[IMX6SL_CLK_EPDC_AXI] = imx_clk_gate2("epdc_axi", "epdc_axi_podf", base + 0x74, 4); - clks[IMX6SL_CLK_LCDIF_AXI] = imx_clk_gate2("lcdif_axi", "lcdif_axi_podf", base + 0x74, 6); - clks[IMX6SL_CLK_LCDIF_PIX] = imx_clk_gate2("lcdif_pix", "lcdif_pix_podf", base + 0x74, 8); - clks[IMX6SL_CLK_EPDC_PIX] = imx_clk_gate2("epdc_pix", "epdc_pix_podf", base + 0x74, 10); - clks[IMX6SL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); - clks[IMX6SL_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26); - clks[IMX6SL_CLK_OCRAM] = imx_clk_gate2("ocram", "ocram_podf", base + 0x74, 28); - clks[IMX6SL_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16); - clks[IMX6SL_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18); - clks[IMX6SL_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20); - clks[IMX6SL_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22); - clks[IMX6SL_CLK_SDMA] = imx_clk_gate2("sdma", "ipg", base + 0x7c, 6); - clks[IMX6SL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12); - clks[IMX6SL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif0_podf", base + 0x7c, 14, &share_count_spdif); - clks[IMX6SL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif); - clks[IMX6SL_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); - clks[IMX6SL_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); - clks[IMX6SL_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); - clks[IMX6SL_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); - clks[IMX6SL_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); - clks[IMX6SL_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); - clks[IMX6SL_CLK_UART] = imx_clk_gate2("uart", "ipg", base + 0x7c, 24); - clks[IMX6SL_CLK_UART_SERIAL] = imx_clk_gate2("uart_serial", "uart_root", base + 0x7c, 26); - clks[IMX6SL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0); - clks[IMX6SL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); - clks[IMX6SL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); - clks[IMX6SL_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); - clks[IMX6SL_CLK_USDHC4] = imx_clk_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8); + hws[IMX6SL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_root", base + 0x6c, 0); + hws[IMX6SL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_root", base + 0x6c, 2); + hws[IMX6SL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_root", base + 0x6c, 4); + hws[IMX6SL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_root", base + 0x6c, 6); + hws[IMX6SL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x6c, 10); + hws[IMX6SL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12); + hws[IMX6SL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14); + hws[IMX6SL_CLK_EXTERN_AUDIO] = imx_clk_hw_gate2("extern_audio", "extern_audio_podf", base + 0x6c, 16); + hws[IMX6SL_CLK_GPT] = imx_clk_hw_gate2("gpt", "perclk", base + 0x6c, 20); + hws[IMX6SL_CLK_GPT_SERIAL] = imx_clk_hw_gate2("gpt_serial", "perclk", base + 0x6c, 22); + hws[IMX6SL_CLK_GPU2D_OVG] = imx_clk_hw_gate2("gpu2d_ovg", "gpu2d_ovg_podf", base + 0x6c, 26); + hws[IMX6SL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6); + hws[IMX6SL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8); + hws[IMX6SL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10); + hws[IMX6SL_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12); + hws[IMX6SL_CLK_CSI] = imx_clk_hw_gate2("csi", "csi_podf", base + 0x74, 0); + hws[IMX6SL_CLK_PXP_AXI] = imx_clk_hw_gate2("pxp_axi", "pxp_axi_podf", base + 0x74, 2); + hws[IMX6SL_CLK_EPDC_AXI] = imx_clk_hw_gate2("epdc_axi", "epdc_axi_podf", base + 0x74, 4); + hws[IMX6SL_CLK_LCDIF_AXI] = imx_clk_hw_gate2("lcdif_axi", "lcdif_axi_podf", base + 0x74, 6); + hws[IMX6SL_CLK_LCDIF_PIX] = imx_clk_hw_gate2("lcdif_pix", "lcdif_pix_podf", base + 0x74, 8); + hws[IMX6SL_CLK_EPDC_PIX] = imx_clk_hw_gate2("epdc_pix", "epdc_pix_podf", base + 0x74, 10); + hws[IMX6SL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); + hws[IMX6SL_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26); + hws[IMX6SL_CLK_OCRAM] = imx_clk_hw_gate2("ocram", "ocram_podf", base + 0x74, 28); + hws[IMX6SL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16); + hws[IMX6SL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18); + hws[IMX6SL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20); + hws[IMX6SL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22); + hws[IMX6SL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ipg", base + 0x7c, 6); + hws[IMX6SL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12); + hws[IMX6SL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif0_podf", base + 0x7c, 14, &share_count_spdif); + hws[IMX6SL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_spdif); + hws[IMX6SL_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6SL_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6SL_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6SL_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6SL_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6SL_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6SL_CLK_UART] = imx_clk_hw_gate2("uart", "ipg", base + 0x7c, 24); + hws[IMX6SL_CLK_UART_SERIAL] = imx_clk_hw_gate2("uart_serial", "uart_root", base + 0x7c, 26); + hws[IMX6SL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0); + hws[IMX6SL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); + hws[IMX6SL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); + hws[IMX6SL_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); + hws[IMX6SL_CLK_USDHC4] = imx_clk_hw_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8); /* Ensure the MMDC CH0 handshake is bypassed */ - writel_relaxed(readl_relaxed(base + CCDR) | - BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR); + imx_mmdc_mask_handshake(base, 0); - imx_check_clocks(clks, ARRAY_SIZE(clks)); + imx_check_clk_hws(hws, IMX6SL_CLK_END); - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); /* Ensure the AHB clk is at 132MHz. */ - ret = clk_set_rate(clks[IMX6SL_CLK_AHB], 132000000); + ret = clk_set_rate(hws[IMX6SL_CLK_AHB]->clk, 132000000); if (ret) pr_warn("%s: failed to set AHB clock rate %d!\n", __func__, ret); if (IS_ENABLED(CONFIG_USB_MXS_PHY)) { - clk_prepare_enable(clks[IMX6SL_CLK_USBPHY1_GATE]); - clk_prepare_enable(clks[IMX6SL_CLK_USBPHY2_GATE]); + clk_prepare_enable(hws[IMX6SL_CLK_USBPHY1_GATE]->clk); + clk_prepare_enable(hws[IMX6SL_CLK_USBPHY2_GATE]->clk); } /* Audio-related clocks configuration */ - clk_set_parent(clks[IMX6SL_CLK_SPDIF0_SEL], clks[IMX6SL_CLK_PLL3_PFD3]); + clk_set_parent(hws[IMX6SL_CLK_SPDIF0_SEL]->clk, hws[IMX6SL_CLK_PLL3_PFD3]->clk); /* set PLL5 video as lcdif pix parent clock */ - clk_set_parent(clks[IMX6SL_CLK_LCDIF_PIX_SEL], - clks[IMX6SL_CLK_PLL5_VIDEO_DIV]); + clk_set_parent(hws[IMX6SL_CLK_LCDIF_PIX_SEL]->clk, + hws[IMX6SL_CLK_PLL5_VIDEO_DIV]->clk); - clk_set_parent(clks[IMX6SL_CLK_LCDIF_AXI_SEL], - clks[IMX6SL_CLK_PLL2_PFD2]); + clk_set_parent(hws[IMX6SL_CLK_LCDIF_AXI_SEL]->clk, + hws[IMX6SL_CLK_PLL2_PFD2]->clk); + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_clks[i] = &hws[index]->clk; + } imx_register_uart_clocks(uart_clks); } diff --git a/drivers/clk/imx/clk-imx6sll.c b/drivers/clk/imx/clk-imx6sll.c index 7eea448cb9a9..5f3e92c09a5e 100644 --- a/drivers/clk/imx/clk-imx6sll.c +++ b/drivers/clk/imx/clk-imx6sll.c @@ -7,6 +7,7 @@ #include <dt-bindings/clock/imx6sll-clock.h> #include <linux/clk.h> #include <linux/clkdev.h> +#include <linux/clk-provider.h> #include <linux/err.h> #include <linux/init.h> #include <linux/io.h> @@ -16,7 +17,6 @@ #include "clk.h" #define CCM_ANALOG_PLL_BYPASS (0x1 << 16) -#define BM_CCM_CCDR_MMDC_CH0_MASK (0x2 << 16) #define xPLL_CLR(offset) (offset + 0x8) static const char *pll_bypass_src_sels[] = { "osc", "dummy", }; @@ -53,8 +53,8 @@ static const char *lcdif_sels[] = { "lcdif_podf", "ipp_di0", "ipp_di1", "ldb_di0 static const char *epdc_pre_sels[] = { "pll2_bus", "pll3_usb_otg", "pll5_video_div", "pll2_pfd0_352m", "pll2_pfd2_396m", "pll3_pfd2_508m", }; static const char *epdc_sels[] = { "epdc_podf", "ipp_di0", "ipp_di1", "ldb_di0", "ldb_di1", }; -static struct clk *clks[IMX6SLL_CLK_END]; -static struct clk_onecell_data clk_data; +static struct clk_hw **hws; +static struct clk_hw_onecell_data *clk_hw_data; static const struct clk_div_table post_div_table[] = { { .val = 2, .div = 1, }, @@ -76,33 +76,43 @@ static u32 share_count_ssi1; static u32 share_count_ssi2; static u32 share_count_ssi3; -static struct clk ** const uart_clks[] __initconst = { - &clks[IMX6SLL_CLK_UART1_IPG], - &clks[IMX6SLL_CLK_UART1_SERIAL], - &clks[IMX6SLL_CLK_UART2_IPG], - &clks[IMX6SLL_CLK_UART2_SERIAL], - &clks[IMX6SLL_CLK_UART3_IPG], - &clks[IMX6SLL_CLK_UART3_SERIAL], - &clks[IMX6SLL_CLK_UART4_IPG], - &clks[IMX6SLL_CLK_UART4_SERIAL], - &clks[IMX6SLL_CLK_UART5_IPG], - &clks[IMX6SLL_CLK_UART5_SERIAL], - NULL +static const int uart_clk_ids[] __initconst = { + IMX6SLL_CLK_UART1_IPG, + IMX6SLL_CLK_UART1_SERIAL, + IMX6SLL_CLK_UART2_IPG, + IMX6SLL_CLK_UART2_SERIAL, + IMX6SLL_CLK_UART3_IPG, + IMX6SLL_CLK_UART3_SERIAL, + IMX6SLL_CLK_UART4_IPG, + IMX6SLL_CLK_UART4_SERIAL, + IMX6SLL_CLK_UART5_IPG, + IMX6SLL_CLK_UART5_SERIAL, }; +static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; + static void __init imx6sll_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; + int i; + + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX6SLL_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return; + + clk_hw_data->num = IMX6SLL_CLK_END; + hws = clk_hw_data->hws; - clks[IMX6SLL_CLK_DUMMY] = imx_clk_fixed("dummy", 0); + hws[IMX6SLL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); - clks[IMX6SLL_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil"); - clks[IMX6SLL_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc"); + hws[IMX6SLL_CLK_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil")); + hws[IMX6SLL_CLK_OSC] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc")); /* ipp_di clock is external input */ - clks[IMX6SLL_CLK_IPP_DI0] = of_clk_get_by_name(ccm_node, "ipp_di0"); - clks[IMX6SLL_CLK_IPP_DI1] = of_clk_get_by_name(ccm_node, "ipp_di1"); + hws[IMX6SLL_CLK_IPP_DI0] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di0")); + hws[IMX6SLL_CLK_IPP_DI1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di1")); np = of_find_compatible_node(NULL, NULL, "fsl,imx6sll-anatop"); base = of_iomap(np, 0); @@ -118,37 +128,37 @@ static void __init imx6sll_clocks_init(struct device_node *ccm_node) writel_relaxed(CCM_ANALOG_PLL_BYPASS, base + xPLL_CLR(0xa0)); writel_relaxed(CCM_ANALOG_PLL_BYPASS, base + xPLL_CLR(0xe0)); - clks[IMX6SLL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SLL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SLL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SLL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SLL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SLL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SLL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - - clks[IMX6SLL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f); - clks[IMX6SLL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1); - clks[IMX6SLL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3); - clks[IMX6SLL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f); - clks[IMX6SLL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f); - clks[IMX6SLL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3); - clks[IMX6SLL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3); - - clks[IMX6SLL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SLL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SLL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SLL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SLL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SLL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SLL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); - - clks[IMX6SLL_CLK_PLL1_SYS] = imx_clk_fixed_factor("pll1_sys", "pll1_bypass", 1, 1); - clks[IMX6SLL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); - clks[IMX6SLL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); - clks[IMX6SLL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); - clks[IMX6SLL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); - clks[IMX6SLL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); - clks[IMX6SLL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); + hws[IMX6SLL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SLL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SLL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SLL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SLL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SLL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SLL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + + hws[IMX6SLL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f); + hws[IMX6SLL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1); + hws[IMX6SLL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3); + hws[IMX6SLL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f); + hws[IMX6SLL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f); + hws[IMX6SLL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3); + hws[IMX6SLL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3); + + hws[IMX6SLL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SLL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SLL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SLL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SLL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SLL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SLL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); + + hws[IMX6SLL_CLK_PLL1_SYS] = imx_clk_hw_fixed_factor("pll1_sys", "pll1_bypass", 1, 1); + hws[IMX6SLL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); + hws[IMX6SLL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); + hws[IMX6SLL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); + hws[IMX6SLL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); + hws[IMX6SLL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); + hws[IMX6SLL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); /* * Bit 20 is the reserved and read-only bit, we do this only for: @@ -156,209 +166,213 @@ static void __init imx6sll_clocks_init(struct device_node *ccm_node) * - Keep refcount when do usbphy clk_enable/disable, in that case, * the clk framework many need to enable/disable usbphy's parent */ - clks[IMX6SLL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); - clks[IMX6SLL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); + hws[IMX6SLL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); + hws[IMX6SLL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); /* * usbphy*_gate needs to be on after system boots up, and software * never needs to control it anymore. */ if (IS_ENABLED(CONFIG_USB_MXS_PHY)) { - clks[IMX6SLL_CLK_USBPHY1_GATE] = imx_clk_gate_flags("usbphy1_gate", "dummy", base + 0x10, 6, CLK_IS_CRITICAL); - clks[IMX6SLL_CLK_USBPHY2_GATE] = imx_clk_gate_flags("usbphy2_gate", "dummy", base + 0x20, 6, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_USBPHY1_GATE] = imx_clk_hw_gate_flags("usbphy1_gate", "dummy", base + 0x10, 6, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_USBPHY2_GATE] = imx_clk_hw_gate_flags("usbphy2_gate", "dummy", base + 0x20, 6, CLK_IS_CRITICAL); } /* name parent_name reg idx */ - clks[IMX6SLL_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); - clks[IMX6SLL_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); - clks[IMX6SLL_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); - clks[IMX6SLL_CLK_PLL2_PFD3] = imx_clk_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3); - clks[IMX6SLL_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); - clks[IMX6SLL_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); - clks[IMX6SLL_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); - clks[IMX6SLL_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); - - clks[IMX6SLL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio", + hws[IMX6SLL_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); + hws[IMX6SLL_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); + hws[IMX6SLL_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); + hws[IMX6SLL_CLK_PLL2_PFD3] = imx_clk_hw_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3); + hws[IMX6SLL_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); + hws[IMX6SLL_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); + hws[IMX6SLL_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); + hws[IMX6SLL_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); + + hws[IMX6SLL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6SLL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div", + hws[IMX6SLL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 15, 1, 0, &imx_ccm_lock); - clks[IMX6SLL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video", + hws[IMX6SLL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6SLL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", + hws[IMX6SLL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); /* name parent_name mult div */ - clks[IMX6SLL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); - clks[IMX6SLL_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); - clks[IMX6SLL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); - clks[IMX6SLL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); + hws[IMX6SLL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); + hws[IMX6SLL_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); + hws[IMX6SLL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); + hws[IMX6SLL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); np = ccm_node; base = of_iomap(np, 0); WARN_ON(!base); - clks[IMX6SLL_CLK_STEP] = imx_clk_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels)); - clks[IMX6SLL_CLK_PLL1_SW] = imx_clk_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0); - clks[IMX6SLL_CLK_AXI_ALT_SEL] = imx_clk_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels)); - clks[IMX6SLL_CLK_AXI_SEL] = imx_clk_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0); - clks[IMX6SLL_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); - clks[IMX6SLL_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels)); - clks[IMX6SLL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); - clks[IMX6SLL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); - clks[IMX6SLL_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6SLL_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6SLL_CLK_USDHC3_SEL] = imx_clk_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6SLL_CLK_SSI1_SEL] = imx_clk_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clks[IMX6SLL_CLK_SSI2_SEL] = imx_clk_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clks[IMX6SLL_CLK_SSI3_SEL] = imx_clk_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clks[IMX6SLL_CLK_PERCLK_SEL] = imx_clk_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels)); - clks[IMX6SLL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); - clks[IMX6SLL_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels)); - clks[IMX6SLL_CLK_EXTERN_AUDIO_SEL] = imx_clk_mux("extern_audio_sel", base + 0x30, 7, 2, spdif_sels, ARRAY_SIZE(spdif_sels)); - clks[IMX6SLL_CLK_EPDC_PRE_SEL] = imx_clk_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels)); - clks[IMX6SLL_CLK_EPDC_SEL] = imx_clk_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels)); - clks[IMX6SLL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); - clks[IMX6SLL_CLK_LCDIF_PRE_SEL] = imx_clk_mux("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels)); - clks[IMX6SLL_CLK_LCDIF_SEL] = imx_clk_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels)); - - clks[IMX6SLL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); - clks[IMX6SLL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); - - clks[IMX6SLL_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); - clks[IMX6SLL_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); - clks[IMX6SLL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2); - clks[IMX6SLL_CLK_LCDIF_PODF] = imx_clk_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3); - clks[IMX6SLL_CLK_PERCLK] = imx_clk_divider("perclk", "perclk_sel", base + 0x1c, 0, 6); - clks[IMX6SLL_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); - clks[IMX6SLL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); - clks[IMX6SLL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); - clks[IMX6SLL_CLK_UART_PODF] = imx_clk_divider("uart_podf", "uart_sel", base + 0x24, 0, 6); - clks[IMX6SLL_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); - clks[IMX6SLL_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); - clks[IMX6SLL_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); - clks[IMX6SLL_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); - clks[IMX6SLL_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); - clks[IMX6SLL_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); - clks[IMX6SLL_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); - clks[IMX6SLL_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); - clks[IMX6SLL_CLK_EXTERN_AUDIO_PRED] = imx_clk_divider("extern_audio_pred", "extern_audio_sel", base + 0x30, 12, 3); - clks[IMX6SLL_CLK_EXTERN_AUDIO_PODF] = imx_clk_divider("extern_audio_podf", "extern_audio_pred", base + 0x30, 9, 3); - clks[IMX6SLL_CLK_EPDC_PODF] = imx_clk_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3); - clks[IMX6SLL_CLK_ECSPI_PODF] = imx_clk_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6); - clks[IMX6SLL_CLK_LCDIF_PRED] = imx_clk_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3); - - clks[IMX6SLL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); - clks[IMX6SLL_CLK_MMDC_PODF] = imx_clk_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); - clks[IMX6SLL_CLK_AXI_PODF] = imx_clk_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0); - clks[IMX6SLL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); - - clks[IMX6SLL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); - clks[IMX6SLL_CLK_LDB_DI0_DIV_7] = imx_clk_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7); - clks[IMX6SLL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7); - clks[IMX6SLL_CLK_LDB_DI1_DIV_7] = imx_clk_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7); - - clks[IMX6SLL_CLK_LDB_DI0_SEL] = imx_clk_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels)); - clks[IMX6SLL_CLK_LDB_DI1_SEL] = imx_clk_mux("ldb_di1_sel", base + 0x1c, 7, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels)); - clks[IMX6SLL_CLK_LDB_DI0_DIV_SEL] = imx_clk_mux("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels)); - clks[IMX6SLL_CLK_LDB_DI1_DIV_SEL] = imx_clk_mux("ldb_di1_div_sel", base + 0x20, 10, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels)); + hws[IMX6SLL_CLK_STEP] = imx_clk_hw_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels)); + hws[IMX6SLL_CLK_PLL1_SW] = imx_clk_hw_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0); + hws[IMX6SLL_CLK_AXI_ALT_SEL] = imx_clk_hw_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels)); + hws[IMX6SLL_CLK_AXI_SEL] = imx_clk_hw_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0); + hws[IMX6SLL_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); + hws[IMX6SLL_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels)); + hws[IMX6SLL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); + hws[IMX6SLL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); + hws[IMX6SLL_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6SLL_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6SLL_CLK_USDHC3_SEL] = imx_clk_hw_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6SLL_CLK_SSI1_SEL] = imx_clk_hw_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6SLL_CLK_SSI2_SEL] = imx_clk_hw_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6SLL_CLK_SSI3_SEL] = imx_clk_hw_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6SLL_CLK_PERCLK_SEL] = imx_clk_hw_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels)); + hws[IMX6SLL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); + hws[IMX6SLL_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels)); + hws[IMX6SLL_CLK_EXTERN_AUDIO_SEL] = imx_clk_hw_mux("extern_audio_sel", base + 0x30, 7, 2, spdif_sels, ARRAY_SIZE(spdif_sels)); + hws[IMX6SLL_CLK_EPDC_PRE_SEL] = imx_clk_hw_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels)); + hws[IMX6SLL_CLK_EPDC_SEL] = imx_clk_hw_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels)); + hws[IMX6SLL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); + hws[IMX6SLL_CLK_LCDIF_PRE_SEL] = imx_clk_hw_mux("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels)); + hws[IMX6SLL_CLK_LCDIF_SEL] = imx_clk_hw_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels)); + + hws[IMX6SLL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); + hws[IMX6SLL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); + + hws[IMX6SLL_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); + hws[IMX6SLL_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); + hws[IMX6SLL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2); + hws[IMX6SLL_CLK_LCDIF_PODF] = imx_clk_hw_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3); + hws[IMX6SLL_CLK_PERCLK] = imx_clk_hw_divider("perclk", "perclk_sel", base + 0x1c, 0, 6); + hws[IMX6SLL_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); + hws[IMX6SLL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); + hws[IMX6SLL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); + hws[IMX6SLL_CLK_UART_PODF] = imx_clk_hw_divider("uart_podf", "uart_sel", base + 0x24, 0, 6); + hws[IMX6SLL_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); + hws[IMX6SLL_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); + hws[IMX6SLL_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); + hws[IMX6SLL_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); + hws[IMX6SLL_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); + hws[IMX6SLL_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); + hws[IMX6SLL_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); + hws[IMX6SLL_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); + hws[IMX6SLL_CLK_EXTERN_AUDIO_PRED] = imx_clk_hw_divider("extern_audio_pred", "extern_audio_sel", base + 0x30, 12, 3); + hws[IMX6SLL_CLK_EXTERN_AUDIO_PODF] = imx_clk_hw_divider("extern_audio_podf", "extern_audio_pred", base + 0x30, 9, 3); + hws[IMX6SLL_CLK_EPDC_PODF] = imx_clk_hw_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3); + hws[IMX6SLL_CLK_ECSPI_PODF] = imx_clk_hw_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6); + hws[IMX6SLL_CLK_LCDIF_PRED] = imx_clk_hw_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3); + + hws[IMX6SLL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); + hws[IMX6SLL_CLK_MMDC_PODF] = imx_clk_hw_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); + hws[IMX6SLL_CLK_AXI_PODF] = imx_clk_hw_busy_divider("axi", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0); + hws[IMX6SLL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); + + hws[IMX6SLL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); + hws[IMX6SLL_CLK_LDB_DI0_DIV_7] = imx_clk_hw_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7); + hws[IMX6SLL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7); + hws[IMX6SLL_CLK_LDB_DI1_DIV_7] = imx_clk_hw_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7); + + hws[IMX6SLL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels)); + hws[IMX6SLL_CLK_LDB_DI1_SEL] = imx_clk_hw_mux("ldb_di1_sel", base + 0x1c, 7, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels)); + hws[IMX6SLL_CLK_LDB_DI0_DIV_SEL] = imx_clk_hw_mux("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels)); + hws[IMX6SLL_CLK_LDB_DI1_DIV_SEL] = imx_clk_hw_mux("ldb_di1_div_sel", base + 0x20, 10, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels)); /* CCGR0 */ - clks[IMX6SLL_CLK_AIPSTZ1] = imx_clk_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL); - clks[IMX6SLL_CLK_AIPSTZ2] = imx_clk_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL); - clks[IMX6SLL_CLK_DCP] = imx_clk_gate2("dcp", "ahb", base + 0x68, 10); - clks[IMX6SLL_CLK_UART2_IPG] = imx_clk_gate2("uart2_ipg", "ipg", base + 0x68, 28); - clks[IMX6SLL_CLK_UART2_SERIAL] = imx_clk_gate2("uart2_serial", "uart_podf", base + 0x68, 28); - clks[IMX6SLL_CLK_GPIO2] = imx_clk_gate2("gpio2", "ipg", base + 0x68, 30); + hws[IMX6SLL_CLK_AIPSTZ1] = imx_clk_hw_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_AIPSTZ2] = imx_clk_hw_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_DCP] = imx_clk_hw_gate2("dcp", "ahb", base + 0x68, 10); + hws[IMX6SLL_CLK_UART2_IPG] = imx_clk_hw_gate2("uart2_ipg", "ipg", base + 0x68, 28); + hws[IMX6SLL_CLK_UART2_SERIAL] = imx_clk_hw_gate2("uart2_serial", "uart_podf", base + 0x68, 28); + hws[IMX6SLL_CLK_GPIO2] = imx_clk_hw_gate2("gpio2", "ipg", base + 0x68, 30); /* CCGR1 */ - clks[IMX6SLL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0); - clks[IMX6SLL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2); - clks[IMX6SLL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4); - clks[IMX6SLL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6); - clks[IMX6SLL_CLK_UART3_IPG] = imx_clk_gate2("uart3_ipg", "ipg", base + 0x6c, 10); - clks[IMX6SLL_CLK_UART3_SERIAL] = imx_clk_gate2("uart3_serial", "uart_podf", base + 0x6c, 10); - clks[IMX6SLL_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12); - clks[IMX6SLL_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14); - clks[IMX6SLL_CLK_GPT_BUS] = imx_clk_gate2("gpt1_bus", "perclk", base + 0x6c, 20); - clks[IMX6SLL_CLK_GPT_SERIAL] = imx_clk_gate2("gpt1_serial", "perclk", base + 0x6c, 22); - clks[IMX6SLL_CLK_UART4_IPG] = imx_clk_gate2("uart4_ipg", "ipg", base + 0x6c, 24); - clks[IMX6SLL_CLK_UART4_SERIAL] = imx_clk_gate2("uart4_serial", "uart_podf", base + 0x6c, 24); - clks[IMX6SLL_CLK_GPIO1] = imx_clk_gate2("gpio1", "ipg", base + 0x6c, 26); - clks[IMX6SLL_CLK_GPIO5] = imx_clk_gate2("gpio5", "ipg", base + 0x6c, 30); + hws[IMX6SLL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0); + hws[IMX6SLL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2); + hws[IMX6SLL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4); + hws[IMX6SLL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6); + hws[IMX6SLL_CLK_UART3_IPG] = imx_clk_hw_gate2("uart3_ipg", "ipg", base + 0x6c, 10); + hws[IMX6SLL_CLK_UART3_SERIAL] = imx_clk_hw_gate2("uart3_serial", "uart_podf", base + 0x6c, 10); + hws[IMX6SLL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12); + hws[IMX6SLL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14); + hws[IMX6SLL_CLK_GPT_BUS] = imx_clk_hw_gate2("gpt1_bus", "perclk", base + 0x6c, 20); + hws[IMX6SLL_CLK_GPT_SERIAL] = imx_clk_hw_gate2("gpt1_serial", "perclk", base + 0x6c, 22); + hws[IMX6SLL_CLK_UART4_IPG] = imx_clk_hw_gate2("uart4_ipg", "ipg", base + 0x6c, 24); + hws[IMX6SLL_CLK_UART4_SERIAL] = imx_clk_hw_gate2("uart4_serial", "uart_podf", base + 0x6c, 24); + hws[IMX6SLL_CLK_GPIO1] = imx_clk_hw_gate2("gpio1", "ipg", base + 0x6c, 26); + hws[IMX6SLL_CLK_GPIO5] = imx_clk_hw_gate2("gpio5", "ipg", base + 0x6c, 30); /* CCGR2 */ - clks[IMX6SLL_CLK_GPIO6] = imx_clk_gate2("gpio6", "ipg", base + 0x70, 0); - clks[IMX6SLL_CLK_CSI] = imx_clk_gate2("csi", "axi", base + 0x70, 2); - clks[IMX6SLL_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6); - clks[IMX6SLL_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8); - clks[IMX6SLL_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10); - clks[IMX6SLL_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12); - clks[IMX6SLL_CLK_GPIO3] = imx_clk_gate2("gpio3", "ipg", base + 0x70, 26); - clks[IMX6SLL_CLK_LCDIF_APB] = imx_clk_gate2("lcdif_apb", "axi", base + 0x70, 28); - clks[IMX6SLL_CLK_PXP] = imx_clk_gate2("pxp", "axi", base + 0x70, 30); + hws[IMX6SLL_CLK_GPIO6] = imx_clk_hw_gate2("gpio6", "ipg", base + 0x70, 0); + hws[IMX6SLL_CLK_CSI] = imx_clk_hw_gate2("csi", "axi", base + 0x70, 2); + hws[IMX6SLL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6); + hws[IMX6SLL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8); + hws[IMX6SLL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10); + hws[IMX6SLL_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12); + hws[IMX6SLL_CLK_GPIO3] = imx_clk_hw_gate2("gpio3", "ipg", base + 0x70, 26); + hws[IMX6SLL_CLK_LCDIF_APB] = imx_clk_hw_gate2("lcdif_apb", "axi", base + 0x70, 28); + hws[IMX6SLL_CLK_PXP] = imx_clk_hw_gate2("pxp", "axi", base + 0x70, 30); /* CCGR3 */ - clks[IMX6SLL_CLK_UART5_IPG] = imx_clk_gate2("uart5_ipg", "ipg", base + 0x74, 2); - clks[IMX6SLL_CLK_UART5_SERIAL] = imx_clk_gate2("uart5_serial", "uart_podf", base + 0x74, 2); - clks[IMX6SLL_CLK_EPDC_AXI] = imx_clk_gate2("epdc_aclk", "axi", base + 0x74, 4); - clks[IMX6SLL_CLK_EPDC_PIX] = imx_clk_gate2("epdc_pix", "epdc_podf", base + 0x74, 4); - clks[IMX6SLL_CLK_LCDIF_PIX] = imx_clk_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10); - clks[IMX6SLL_CLK_GPIO4] = imx_clk_gate2("gpio4", "ipg", base + 0x74, 12); - clks[IMX6SLL_CLK_WDOG1] = imx_clk_gate2("wdog1", "ipg", base + 0x74, 16); - clks[IMX6SLL_CLK_MMDC_P0_FAST] = imx_clk_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL); - clks[IMX6SLL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); - clks[IMX6SLL_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26); - clks[IMX6SLL_CLK_OCRAM] = imx_clk_gate_flags("ocram","ahb", base + 0x74, 28, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_UART5_IPG] = imx_clk_hw_gate2("uart5_ipg", "ipg", base + 0x74, 2); + hws[IMX6SLL_CLK_UART5_SERIAL] = imx_clk_hw_gate2("uart5_serial", "uart_podf", base + 0x74, 2); + hws[IMX6SLL_CLK_EPDC_AXI] = imx_clk_hw_gate2("epdc_aclk", "axi", base + 0x74, 4); + hws[IMX6SLL_CLK_EPDC_PIX] = imx_clk_hw_gate2("epdc_pix", "epdc_podf", base + 0x74, 4); + hws[IMX6SLL_CLK_LCDIF_PIX] = imx_clk_hw_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10); + hws[IMX6SLL_CLK_GPIO4] = imx_clk_hw_gate2("gpio4", "ipg", base + 0x74, 12); + hws[IMX6SLL_CLK_WDOG1] = imx_clk_hw_gate2("wdog1", "ipg", base + 0x74, 16); + hws[IMX6SLL_CLK_MMDC_P0_FAST] = imx_clk_hw_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2_flags("mmdc_p1_ipg", "ipg", base + 0x74, 26, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_OCRAM] = imx_clk_hw_gate_flags("ocram", "ahb", base + 0x74, 28, CLK_IS_CRITICAL); /* CCGR4 */ - clks[IMX6SLL_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16); - clks[IMX6SLL_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18); - clks[IMX6SLL_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20); - clks[IMX6SLL_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22); + hws[IMX6SLL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16); + hws[IMX6SLL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18); + hws[IMX6SLL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20); + hws[IMX6SLL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22); /* CCGR5 */ - clks[IMX6SLL_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); - clks[IMX6SLL_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6); - clks[IMX6SLL_CLK_WDOG2] = imx_clk_gate2("wdog2", "ipg", base + 0x7c, 10); - clks[IMX6SLL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12); - clks[IMX6SLL_CLK_EXTERN_AUDIO] = imx_clk_gate2_shared("extern_audio", "extern_audio_podf", base + 0x7c, 14, &share_count_audio); - clks[IMX6SLL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio); - clks[IMX6SLL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio); - clks[IMX6SLL_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); - clks[IMX6SLL_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); - clks[IMX6SLL_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); - clks[IMX6SLL_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); - clks[IMX6SLL_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); - clks[IMX6SLL_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); - clks[IMX6SLL_CLK_UART1_IPG] = imx_clk_gate2("uart1_ipg", "ipg", base + 0x7c, 24); - clks[IMX6SLL_CLK_UART1_SERIAL] = imx_clk_gate2("uart1_serial", "uart_podf", base + 0x7c, 24); + hws[IMX6SLL_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); + hws[IMX6SLL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6); + hws[IMX6SLL_CLK_WDOG2] = imx_clk_hw_gate2("wdog2", "ipg", base + 0x7c, 10); + hws[IMX6SLL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12); + hws[IMX6SLL_CLK_EXTERN_AUDIO] = imx_clk_hw_gate2_shared("extern_audio", "extern_audio_podf", base + 0x7c, 14, &share_count_audio); + hws[IMX6SLL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio); + hws[IMX6SLL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio); + hws[IMX6SLL_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6SLL_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6SLL_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6SLL_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6SLL_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6SLL_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6SLL_CLK_UART1_IPG] = imx_clk_hw_gate2("uart1_ipg", "ipg", base + 0x7c, 24); + hws[IMX6SLL_CLK_UART1_SERIAL] = imx_clk_hw_gate2("uart1_serial", "uart_podf", base + 0x7c, 24); /* CCGR6 */ - clks[IMX6SLL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0); - clks[IMX6SLL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); - clks[IMX6SLL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); - clks[IMX6SLL_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); + hws[IMX6SLL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0); + hws[IMX6SLL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); + hws[IMX6SLL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); + hws[IMX6SLL_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); /* mask handshake of mmdc */ - writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + 0x4); + imx_mmdc_mask_handshake(base, 0); - imx_check_clocks(clks, ARRAY_SIZE(clks)); + imx_check_clk_hws(hws, IMX6SLL_CLK_END); - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_clks[i] = &hws[index]->clk; + } imx_register_uart_clocks(uart_clks); /* Lower the AHB clock rate before changing the clock source. */ - clk_set_rate(clks[IMX6SLL_CLK_AHB], 99000000); + clk_set_rate(hws[IMX6SLL_CLK_AHB]->clk, 99000000); /* Change periph_pre clock to pll2_bus to adjust AXI rate to 264MHz */ - clk_set_parent(clks[IMX6SLL_CLK_PERIPH_CLK2_SEL], clks[IMX6SLL_CLK_PLL3_USB_OTG]); - clk_set_parent(clks[IMX6SLL_CLK_PERIPH], clks[IMX6SLL_CLK_PERIPH_CLK2]); - clk_set_parent(clks[IMX6SLL_CLK_PERIPH_PRE], clks[IMX6SLL_CLK_PLL2_BUS]); - clk_set_parent(clks[IMX6SLL_CLK_PERIPH], clks[IMX6SLL_CLK_PERIPH_PRE]); + clk_set_parent(hws[IMX6SLL_CLK_PERIPH_CLK2_SEL]->clk, hws[IMX6SLL_CLK_PLL3_USB_OTG]->clk); + clk_set_parent(hws[IMX6SLL_CLK_PERIPH]->clk, hws[IMX6SLL_CLK_PERIPH_CLK2]->clk); + clk_set_parent(hws[IMX6SLL_CLK_PERIPH_PRE]->clk, hws[IMX6SLL_CLK_PLL2_BUS]->clk); + clk_set_parent(hws[IMX6SLL_CLK_PERIPH]->clk, hws[IMX6SLL_CLK_PERIPH_PRE]->clk); - clk_set_rate(clks[IMX6SLL_CLK_AHB], 132000000); + clk_set_rate(hws[IMX6SLL_CLK_AHB]->clk, 132000000); } CLK_OF_DECLARE_DRIVER(imx6sll, "fsl,imx6sll-ccm", imx6sll_clocks_init); diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c index d243e3483e24..c4685c01929a 100644 --- a/drivers/clk/imx/clk-imx6sx.c +++ b/drivers/clk/imx/clk-imx6sx.c @@ -6,6 +6,7 @@ #include <dt-bindings/clock/imx6sx-clock.h> #include <linux/clk.h> #include <linux/clkdev.h> +#include <linux/clk-provider.h> #include <linux/err.h> #include <linux/init.h> #include <linux/io.h> @@ -16,9 +17,6 @@ #include "clk.h" -#define CCDR 0x4 -#define BM_CCM_CCDR_MMDC_CH0_MASK (0x2 << 16) - static const char *step_sels[] = { "osc", "pll2_pfd2_396m", }; static const char *pll1_sw_sels[] = { "pll1_sys", "step", }; static const char *periph_pre_sels[] = { "pll2_bus", "pll2_pfd2_396m", "pll2_pfd0_352m", "pll2_198m", }; @@ -83,8 +81,8 @@ static const char *pll5_bypass_sels[] = { "pll5", "pll5_bypass_src", }; static const char *pll6_bypass_sels[] = { "pll6", "pll6_bypass_src", }; static const char *pll7_bypass_sels[] = { "pll7", "pll7_bypass_src", }; -static struct clk *clks[IMX6SX_CLK_CLK_END]; -static struct clk_onecell_data clk_data; +static struct clk_hw **hws; +static struct clk_hw_onecell_data *clk_hw_data; static const struct clk_div_table clk_enet_ref_table[] = { { .val = 0, .div = 20, }, @@ -118,76 +116,86 @@ static u32 share_count_ssi3; static u32 share_count_sai1; static u32 share_count_sai2; -static struct clk ** const uart_clks[] __initconst = { - &clks[IMX6SX_CLK_UART_IPG], - &clks[IMX6SX_CLK_UART_SERIAL], - NULL +static const int uart_clk_ids[] __initconst = { + IMX6SX_CLK_UART_IPG, + IMX6SX_CLK_UART_SERIAL, }; +static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; + static void __init imx6sx_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; + int i; + + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX6SX_CLK_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return; + + clk_hw_data->num = IMX6SX_CLK_CLK_END; + hws = clk_hw_data->hws; - clks[IMX6SX_CLK_DUMMY] = imx_clk_fixed("dummy", 0); + hws[IMX6SX_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); - clks[IMX6SX_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil"); - clks[IMX6SX_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc"); + hws[IMX6SX_CLK_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil")); + hws[IMX6SX_CLK_OSC] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc")); /* ipp_di clock is external input */ - clks[IMX6SX_CLK_IPP_DI0] = of_clk_get_by_name(ccm_node, "ipp_di0"); - clks[IMX6SX_CLK_IPP_DI1] = of_clk_get_by_name(ccm_node, "ipp_di1"); + hws[IMX6SX_CLK_IPP_DI0] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di0")); + hws[IMX6SX_CLK_IPP_DI1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di1")); /* Clock source from external clock via CLK1/2 PAD */ - clks[IMX6SX_CLK_ANACLK1] = of_clk_get_by_name(ccm_node, "anaclk1"); - clks[IMX6SX_CLK_ANACLK2] = of_clk_get_by_name(ccm_node, "anaclk2"); + hws[IMX6SX_CLK_ANACLK1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "anaclk1")); + hws[IMX6SX_CLK_ANACLK2] = __clk_get_hw(of_clk_get_by_name(ccm_node, "anaclk2")); np = of_find_compatible_node(NULL, NULL, "fsl,imx6sx-anatop"); base = of_iomap(np, 0); WARN_ON(!base); of_node_put(np); - clks[IMX6SX_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SX_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SX_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SX_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SX_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SX_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6SX_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SX_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SX_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SX_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SX_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SX_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SX_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6SX_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); /* type name parent_name base div_mask */ - clks[IMX6SX_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); - clks[IMX6SX_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); - clks[IMX6SX_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); - clks[IMX6SX_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); - clks[IMX6SX_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); - clks[IMX6SX_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); - clks[IMX6SX_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); - - clks[IMX6SX_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + hws[IMX6SX_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + hws[IMX6SX_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + hws[IMX6SX_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + hws[IMX6SX_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + hws[IMX6SX_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + hws[IMX6SX_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); + + hws[IMX6SX_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); /* Do not bypass PLLs initially */ - clk_set_parent(clks[IMX6SX_PLL1_BYPASS], clks[IMX6SX_CLK_PLL1]); - clk_set_parent(clks[IMX6SX_PLL2_BYPASS], clks[IMX6SX_CLK_PLL2]); - clk_set_parent(clks[IMX6SX_PLL3_BYPASS], clks[IMX6SX_CLK_PLL3]); - clk_set_parent(clks[IMX6SX_PLL4_BYPASS], clks[IMX6SX_CLK_PLL4]); - clk_set_parent(clks[IMX6SX_PLL5_BYPASS], clks[IMX6SX_CLK_PLL5]); - clk_set_parent(clks[IMX6SX_PLL6_BYPASS], clks[IMX6SX_CLK_PLL6]); - clk_set_parent(clks[IMX6SX_PLL7_BYPASS], clks[IMX6SX_CLK_PLL7]); - - clks[IMX6SX_CLK_PLL1_SYS] = imx_clk_gate("pll1_sys", "pll1_bypass", base + 0x00, 13); - clks[IMX6SX_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); - clks[IMX6SX_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); - clks[IMX6SX_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); - clks[IMX6SX_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); - clks[IMX6SX_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); - clks[IMX6SX_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); + clk_set_parent(hws[IMX6SX_PLL1_BYPASS]->clk, hws[IMX6SX_CLK_PLL1]->clk); + clk_set_parent(hws[IMX6SX_PLL2_BYPASS]->clk, hws[IMX6SX_CLK_PLL2]->clk); + clk_set_parent(hws[IMX6SX_PLL3_BYPASS]->clk, hws[IMX6SX_CLK_PLL3]->clk); + clk_set_parent(hws[IMX6SX_PLL4_BYPASS]->clk, hws[IMX6SX_CLK_PLL4]->clk); + clk_set_parent(hws[IMX6SX_PLL5_BYPASS]->clk, hws[IMX6SX_CLK_PLL5]->clk); + clk_set_parent(hws[IMX6SX_PLL6_BYPASS]->clk, hws[IMX6SX_CLK_PLL6]->clk); + clk_set_parent(hws[IMX6SX_PLL7_BYPASS]->clk, hws[IMX6SX_CLK_PLL7]->clk); + + hws[IMX6SX_CLK_PLL1_SYS] = imx_clk_hw_gate("pll1_sys", "pll1_bypass", base + 0x00, 13); + hws[IMX6SX_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); + hws[IMX6SX_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); + hws[IMX6SX_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); + hws[IMX6SX_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); + hws[IMX6SX_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); + hws[IMX6SX_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); /* * Bit 20 is the reserved and read-only bit, we do this only for: @@ -195,361 +203,363 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node) * - Keep refcount when do usbphy clk_enable/disable, in that case, * the clk framework may need to enable/disable usbphy's parent */ - clks[IMX6SX_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); - clks[IMX6SX_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); + hws[IMX6SX_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); + hws[IMX6SX_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); /* * usbphy*_gate needs to be on after system boots up, and software * never needs to control it anymore. */ - clks[IMX6SX_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6); - clks[IMX6SX_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6); + hws[IMX6SX_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6); + hws[IMX6SX_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6); /* FIXME 100MHz is used for pcie ref for all imx6 pcie, excepted imx6q */ - clks[IMX6SX_CLK_PCIE_REF] = imx_clk_fixed_factor("pcie_ref", "pll6_enet", 1, 5); - clks[IMX6SX_CLK_PCIE_REF_125M] = imx_clk_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19); + hws[IMX6SX_CLK_PCIE_REF] = imx_clk_hw_fixed_factor("pcie_ref", "pll6_enet", 1, 5); + hws[IMX6SX_CLK_PCIE_REF_125M] = imx_clk_hw_gate("pcie_ref_125m", "pcie_ref", base + 0xe0, 19); - clks[IMX6SX_CLK_LVDS1_OUT] = imx_clk_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12)); - clks[IMX6SX_CLK_LVDS2_OUT] = imx_clk_gate_exclusive("lvds2_out", "lvds2_sel", base + 0x160, 11, BIT(13)); - clks[IMX6SX_CLK_LVDS1_IN] = imx_clk_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10)); - clks[IMX6SX_CLK_LVDS2_IN] = imx_clk_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11)); + hws[IMX6SX_CLK_LVDS1_OUT] = imx_clk_hw_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x160, 10, BIT(12)); + hws[IMX6SX_CLK_LVDS2_OUT] = imx_clk_hw_gate_exclusive("lvds2_out", "lvds2_sel", base + 0x160, 11, BIT(13)); + hws[IMX6SX_CLK_LVDS1_IN] = imx_clk_hw_gate_exclusive("lvds1_in", "anaclk1", base + 0x160, 12, BIT(10)); + hws[IMX6SX_CLK_LVDS2_IN] = imx_clk_hw_gate_exclusive("lvds2_in", "anaclk2", base + 0x160, 13, BIT(11)); - clks[IMX6SX_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, + hws[IMX6SX_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock); - clks[IMX6SX_CLK_ENET2_REF] = clk_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0, + hws[IMX6SX_CLK_ENET2_REF] = clk_hw_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0, base + 0xe0, 2, 2, 0, clk_enet_ref_table, &imx_ccm_lock); - clks[IMX6SX_CLK_ENET2_REF_125M] = imx_clk_gate("enet2_ref_125m", "enet2_ref", base + 0xe0, 20); + hws[IMX6SX_CLK_ENET2_REF_125M] = imx_clk_hw_gate("enet2_ref_125m", "enet2_ref", base + 0xe0, 20); - clks[IMX6SX_CLK_ENET_PTP_REF] = imx_clk_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20); - clks[IMX6SX_CLK_ENET_PTP] = imx_clk_gate("enet_ptp_25m", "enet_ptp_ref", base + 0xe0, 21); + hws[IMX6SX_CLK_ENET_PTP_REF] = imx_clk_hw_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20); + hws[IMX6SX_CLK_ENET_PTP] = imx_clk_hw_gate("enet_ptp_25m", "enet_ptp_ref", base + 0xe0, 21); /* name parent_name reg idx */ - clks[IMX6SX_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); - clks[IMX6SX_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); - clks[IMX6SX_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); - clks[IMX6SX_CLK_PLL2_PFD3] = imx_clk_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3); - clks[IMX6SX_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); - clks[IMX6SX_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); - clks[IMX6SX_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); - clks[IMX6SX_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); + hws[IMX6SX_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); + hws[IMX6SX_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); + hws[IMX6SX_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); + hws[IMX6SX_CLK_PLL2_PFD3] = imx_clk_hw_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3); + hws[IMX6SX_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); + hws[IMX6SX_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); + hws[IMX6SX_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); + hws[IMX6SX_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); /* name parent_name mult div */ - clks[IMX6SX_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); - clks[IMX6SX_CLK_PLL3_120M] = imx_clk_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); - clks[IMX6SX_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); - clks[IMX6SX_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); - clks[IMX6SX_CLK_TWD] = imx_clk_fixed_factor("twd", "arm", 1, 2); - clks[IMX6SX_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8); - - clks[IMX6SX_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio", + hws[IMX6SX_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); + hws[IMX6SX_CLK_PLL3_120M] = imx_clk_hw_fixed_factor("pll3_120m", "pll3_usb_otg", 1, 4); + hws[IMX6SX_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); + hws[IMX6SX_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); + hws[IMX6SX_CLK_TWD] = imx_clk_hw_fixed_factor("twd", "arm", 1, 2); + hws[IMX6SX_CLK_GPT_3M] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8); + + hws[IMX6SX_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6SX_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div", + hws[IMX6SX_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT, base + 0x170, 15, 1, 0, &imx_ccm_lock); - clks[IMX6SX_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video", + hws[IMX6SX_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6SX_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", + hws[IMX6SX_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); /* name reg shift width parent_names num_parents */ - clks[IMX6SX_CLK_LVDS1_SEL] = imx_clk_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); - clks[IMX6SX_CLK_LVDS2_SEL] = imx_clk_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); + hws[IMX6SX_CLK_LVDS1_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x160, 0, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); + hws[IMX6SX_CLK_LVDS2_SEL] = imx_clk_hw_mux("lvds2_sel", base + 0x160, 5, 5, lvds_sels, ARRAY_SIZE(lvds_sels)); np = ccm_node; base = of_iomap(np, 0); WARN_ON(!base); /* name reg shift width parent_names num_parents */ - clks[IMX6SX_CLK_STEP] = imx_clk_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels)); - clks[IMX6SX_CLK_PLL1_SW] = imx_clk_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels)); - clks[IMX6SX_CLK_OCRAM_SEL] = imx_clk_mux("ocram_sel", base + 0x14, 6, 2, ocram_sels, ARRAY_SIZE(ocram_sels)); - clks[IMX6SX_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); - clks[IMX6SX_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels)); - clks[IMX6SX_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); - clks[IMX6SX_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); - clks[IMX6SX_CLK_PCIE_AXI_SEL] = imx_clk_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels)); - clks[IMX6SX_CLK_GPU_AXI_SEL] = imx_clk_mux("gpu_axi_sel", base + 0x18, 8, 2, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels)); - clks[IMX6SX_CLK_GPU_CORE_SEL] = imx_clk_mux("gpu_core_sel", base + 0x18, 4, 2, gpu_core_sels, ARRAY_SIZE(gpu_core_sels)); - clks[IMX6SX_CLK_EIM_SLOW_SEL] = imx_clk_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels)); - clks[IMX6SX_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6SX_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6SX_CLK_USDHC3_SEL] = imx_clk_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6SX_CLK_USDHC4_SEL] = imx_clk_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6SX_CLK_SSI3_SEL] = imx_clk_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clks[IMX6SX_CLK_SSI2_SEL] = imx_clk_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clks[IMX6SX_CLK_SSI1_SEL] = imx_clk_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - clks[IMX6SX_CLK_QSPI1_SEL] = imx_clk_mux_flags("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_CLK_PERCLK_SEL] = imx_clk_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels)); - clks[IMX6SX_CLK_VID_SEL] = imx_clk_mux("vid_sel", base + 0x20, 21, 3, vid_sels, ARRAY_SIZE(vid_sels)); - clks[IMX6SX_CLK_ESAI_SEL] = imx_clk_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clks[IMX6SX_CLK_CAN_SEL] = imx_clk_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels)); - clks[IMX6SX_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); - clks[IMX6SX_CLK_QSPI2_SEL] = imx_clk_mux_flags("qspi2_sel", base + 0x2c, 15, 3, qspi2_sels, ARRAY_SIZE(qspi2_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clks[IMX6SX_CLK_AUDIO_SEL] = imx_clk_mux("audio_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels)); - clks[IMX6SX_CLK_ENET_PRE_SEL] = imx_clk_mux("enet_pre_sel", base + 0x34, 15, 3, enet_pre_sels, ARRAY_SIZE(enet_pre_sels)); - clks[IMX6SX_CLK_ENET_SEL] = imx_clk_mux("enet_sel", base + 0x34, 9, 3, enet_sels, ARRAY_SIZE(enet_sels)); - clks[IMX6SX_CLK_M4_PRE_SEL] = imx_clk_mux("m4_pre_sel", base + 0x34, 6, 3, m4_pre_sels, ARRAY_SIZE(m4_pre_sels)); - clks[IMX6SX_CLK_M4_SEL] = imx_clk_mux("m4_sel", base + 0x34, 0, 3, m4_sels, ARRAY_SIZE(m4_sels)); - clks[IMX6SX_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); - clks[IMX6SX_CLK_LCDIF2_PRE_SEL] = imx_clk_mux("lcdif2_pre_sel", base + 0x38, 6, 3, lcdif2_pre_sels, ARRAY_SIZE(lcdif2_pre_sels)); - clks[IMX6SX_CLK_LCDIF2_SEL] = imx_clk_mux("lcdif2_sel", base + 0x38, 0, 3, lcdif2_sels, ARRAY_SIZE(lcdif2_sels)); - clks[IMX6SX_CLK_DISPLAY_SEL] = imx_clk_mux("display_sel", base + 0x3c, 14, 2, display_sels, ARRAY_SIZE(display_sels)); - clks[IMX6SX_CLK_CSI_SEL] = imx_clk_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels)); - clks[IMX6SX_CLK_CKO1_SEL] = imx_clk_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels)); - clks[IMX6SX_CLK_CKO2_SEL] = imx_clk_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels)); - clks[IMX6SX_CLK_CKO] = imx_clk_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels)); - - clks[IMX6SX_CLK_LDB_DI1_DIV_SEL] = imx_clk_mux_flags("ldb_di1_div_sel", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_CLK_LDB_DI0_DIV_SEL] = imx_clk_mux_flags("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_CLK_LDB_DI1_SEL] = imx_clk_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_CLK_LDB_DI0_SEL] = imx_clk_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_CLK_LCDIF1_PRE_SEL] = imx_clk_mux_flags("lcdif1_pre_sel", base + 0x38, 15, 3, lcdif1_pre_sels, ARRAY_SIZE(lcdif1_pre_sels), CLK_SET_RATE_PARENT); - clks[IMX6SX_CLK_LCDIF1_SEL] = imx_clk_mux_flags("lcdif1_sel", base + 0x38, 9, 3, lcdif1_sels, ARRAY_SIZE(lcdif1_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_STEP] = imx_clk_hw_mux("step", base + 0xc, 8, 1, step_sels, ARRAY_SIZE(step_sels)); + hws[IMX6SX_CLK_PLL1_SW] = imx_clk_hw_mux("pll1_sw", base + 0xc, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels)); + hws[IMX6SX_CLK_OCRAM_SEL] = imx_clk_hw_mux("ocram_sel", base + 0x14, 6, 2, ocram_sels, ARRAY_SIZE(ocram_sels)); + hws[IMX6SX_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); + hws[IMX6SX_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels)); + hws[IMX6SX_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); + hws[IMX6SX_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); + hws[IMX6SX_CLK_PCIE_AXI_SEL] = imx_clk_hw_mux("pcie_axi_sel", base + 0x18, 10, 1, pcie_axi_sels, ARRAY_SIZE(pcie_axi_sels)); + hws[IMX6SX_CLK_GPU_AXI_SEL] = imx_clk_hw_mux("gpu_axi_sel", base + 0x18, 8, 2, gpu_axi_sels, ARRAY_SIZE(gpu_axi_sels)); + hws[IMX6SX_CLK_GPU_CORE_SEL] = imx_clk_hw_mux("gpu_core_sel", base + 0x18, 4, 2, gpu_core_sels, ARRAY_SIZE(gpu_core_sels)); + hws[IMX6SX_CLK_EIM_SLOW_SEL] = imx_clk_hw_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels)); + hws[IMX6SX_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6SX_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6SX_CLK_USDHC3_SEL] = imx_clk_hw_mux("usdhc3_sel", base + 0x1c, 18, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6SX_CLK_USDHC4_SEL] = imx_clk_hw_mux("usdhc4_sel", base + 0x1c, 19, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6SX_CLK_SSI3_SEL] = imx_clk_hw_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6SX_CLK_SSI2_SEL] = imx_clk_hw_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6SX_CLK_SSI1_SEL] = imx_clk_hw_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); + hws[IMX6SX_CLK_QSPI1_SEL] = imx_clk_hw_mux_flags("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_PERCLK_SEL] = imx_clk_hw_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels)); + hws[IMX6SX_CLK_VID_SEL] = imx_clk_hw_mux("vid_sel", base + 0x20, 21, 3, vid_sels, ARRAY_SIZE(vid_sels)); + hws[IMX6SX_CLK_ESAI_SEL] = imx_clk_hw_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6SX_CLK_CAN_SEL] = imx_clk_hw_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels)); + hws[IMX6SX_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); + hws[IMX6SX_CLK_QSPI2_SEL] = imx_clk_hw_mux_flags("qspi2_sel", base + 0x2c, 15, 3, qspi2_sels, ARRAY_SIZE(qspi2_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6SX_CLK_AUDIO_SEL] = imx_clk_hw_mux("audio_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels)); + hws[IMX6SX_CLK_ENET_PRE_SEL] = imx_clk_hw_mux("enet_pre_sel", base + 0x34, 15, 3, enet_pre_sels, ARRAY_SIZE(enet_pre_sels)); + hws[IMX6SX_CLK_ENET_SEL] = imx_clk_hw_mux("enet_sel", base + 0x34, 9, 3, enet_sels, ARRAY_SIZE(enet_sels)); + hws[IMX6SX_CLK_M4_PRE_SEL] = imx_clk_hw_mux("m4_pre_sel", base + 0x34, 6, 3, m4_pre_sels, ARRAY_SIZE(m4_pre_sels)); + hws[IMX6SX_CLK_M4_SEL] = imx_clk_hw_mux("m4_sel", base + 0x34, 0, 3, m4_sels, ARRAY_SIZE(m4_sels)); + hws[IMX6SX_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); + hws[IMX6SX_CLK_LCDIF2_PRE_SEL] = imx_clk_hw_mux("lcdif2_pre_sel", base + 0x38, 6, 3, lcdif2_pre_sels, ARRAY_SIZE(lcdif2_pre_sels)); + hws[IMX6SX_CLK_LCDIF2_SEL] = imx_clk_hw_mux("lcdif2_sel", base + 0x38, 0, 3, lcdif2_sels, ARRAY_SIZE(lcdif2_sels)); + hws[IMX6SX_CLK_DISPLAY_SEL] = imx_clk_hw_mux("display_sel", base + 0x3c, 14, 2, display_sels, ARRAY_SIZE(display_sels)); + hws[IMX6SX_CLK_CSI_SEL] = imx_clk_hw_mux("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels)); + hws[IMX6SX_CLK_CKO1_SEL] = imx_clk_hw_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels)); + hws[IMX6SX_CLK_CKO2_SEL] = imx_clk_hw_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels)); + hws[IMX6SX_CLK_CKO] = imx_clk_hw_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels)); + + hws[IMX6SX_CLK_LDB_DI1_DIV_SEL] = imx_clk_hw_mux_flags("ldb_di1_div_sel", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_LDB_DI0_DIV_SEL] = imx_clk_hw_mux_flags("ldb_di0_div_sel", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_LDB_DI1_SEL] = imx_clk_hw_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di1_sels, ARRAY_SIZE(ldb_di1_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_LDB_DI0_SEL] = imx_clk_hw_mux_flags("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_LCDIF1_PRE_SEL] = imx_clk_hw_mux_flags("lcdif1_pre_sel", base + 0x38, 15, 3, lcdif1_pre_sels, ARRAY_SIZE(lcdif1_pre_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_LCDIF1_SEL] = imx_clk_hw_mux_flags("lcdif1_sel", base + 0x38, 9, 3, lcdif1_sels, ARRAY_SIZE(lcdif1_sels), CLK_SET_RATE_PARENT); /* name parent_name reg shift width */ - clks[IMX6SX_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); - clks[IMX6SX_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); - clks[IMX6SX_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2); - clks[IMX6SX_CLK_GPU_CORE_PODF] = imx_clk_divider("gpu_core_podf", "gpu_core_sel", base + 0x18, 29, 3); - clks[IMX6SX_CLK_GPU_AXI_PODF] = imx_clk_divider("gpu_axi_podf", "gpu_axi_sel", base + 0x18, 26, 3); - clks[IMX6SX_CLK_LCDIF1_PODF] = imx_clk_divider("lcdif1_podf", "lcdif1_pred", base + 0x18, 23, 3); - clks[IMX6SX_CLK_QSPI1_PODF] = imx_clk_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3); - clks[IMX6SX_CLK_EIM_SLOW_PODF] = imx_clk_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3); - clks[IMX6SX_CLK_LCDIF2_PODF] = imx_clk_divider("lcdif2_podf", "lcdif2_pred", base + 0x1c, 20, 3); - clks[IMX6SX_CLK_PERCLK] = imx_clk_divider_flags("perclk", "perclk_sel", base + 0x1c, 0, 6, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_VID_PODF] = imx_clk_divider("vid_podf", "vid_sel", base + 0x20, 24, 2); - clks[IMX6SX_CLK_CAN_PODF] = imx_clk_divider("can_podf", "can_sel", base + 0x20, 2, 6); - clks[IMX6SX_CLK_USDHC4_PODF] = imx_clk_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3); - clks[IMX6SX_CLK_USDHC3_PODF] = imx_clk_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); - clks[IMX6SX_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); - clks[IMX6SX_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); - clks[IMX6SX_CLK_UART_PODF] = imx_clk_divider("uart_podf", "uart_sel", base + 0x24, 0, 6); - clks[IMX6SX_CLK_ESAI_PRED] = imx_clk_divider("esai_pred", "esai_sel", base + 0x28, 9, 3); - clks[IMX6SX_CLK_ESAI_PODF] = imx_clk_divider("esai_podf", "esai_pred", base + 0x28, 25, 3); - clks[IMX6SX_CLK_SSI3_PRED] = imx_clk_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); - clks[IMX6SX_CLK_SSI3_PODF] = imx_clk_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); - clks[IMX6SX_CLK_SSI1_PRED] = imx_clk_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); - clks[IMX6SX_CLK_SSI1_PODF] = imx_clk_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); - clks[IMX6SX_CLK_QSPI2_PRED] = imx_clk_divider("qspi2_pred", "qspi2_sel", base + 0x2c, 18, 3); - clks[IMX6SX_CLK_QSPI2_PODF] = imx_clk_divider("qspi2_podf", "qspi2_pred", base + 0x2c, 21, 6); - clks[IMX6SX_CLK_SSI2_PRED] = imx_clk_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); - clks[IMX6SX_CLK_SSI2_PODF] = imx_clk_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); - clks[IMX6SX_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); - clks[IMX6SX_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); - clks[IMX6SX_CLK_AUDIO_PRED] = imx_clk_divider("audio_pred", "audio_sel", base + 0x30, 12, 3); - clks[IMX6SX_CLK_AUDIO_PODF] = imx_clk_divider("audio_podf", "audio_pred", base + 0x30, 9, 3); - clks[IMX6SX_CLK_ENET_PODF] = imx_clk_divider("enet_podf", "enet_pre_sel", base + 0x34, 12, 3); - clks[IMX6SX_CLK_M4_PODF] = imx_clk_divider("m4_podf", "m4_sel", base + 0x34, 3, 3); - clks[IMX6SX_CLK_ECSPI_PODF] = imx_clk_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6); - clks[IMX6SX_CLK_LCDIF1_PRED] = imx_clk_divider("lcdif1_pred", "lcdif1_pre_sel", base + 0x38, 12, 3); - clks[IMX6SX_CLK_LCDIF2_PRED] = imx_clk_divider("lcdif2_pred", "lcdif2_pre_sel", base + 0x38, 3, 3); - clks[IMX6SX_CLK_DISPLAY_PODF] = imx_clk_divider("display_podf", "display_sel", base + 0x3c, 16, 3); - clks[IMX6SX_CLK_CSI_PODF] = imx_clk_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3); - clks[IMX6SX_CLK_CKO1_PODF] = imx_clk_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3); - clks[IMX6SX_CLK_CKO2_PODF] = imx_clk_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3); - - clks[IMX6SX_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); - clks[IMX6SX_CLK_LDB_DI0_DIV_7] = imx_clk_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7); - clks[IMX6SX_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7); - clks[IMX6SX_CLK_LDB_DI1_DIV_7] = imx_clk_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7); + hws[IMX6SX_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); + hws[IMX6SX_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); + hws[IMX6SX_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2); + hws[IMX6SX_CLK_GPU_CORE_PODF] = imx_clk_hw_divider("gpu_core_podf", "gpu_core_sel", base + 0x18, 29, 3); + hws[IMX6SX_CLK_GPU_AXI_PODF] = imx_clk_hw_divider("gpu_axi_podf", "gpu_axi_sel", base + 0x18, 26, 3); + hws[IMX6SX_CLK_LCDIF1_PODF] = imx_clk_hw_divider("lcdif1_podf", "lcdif1_pred", base + 0x18, 23, 3); + hws[IMX6SX_CLK_QSPI1_PODF] = imx_clk_hw_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3); + hws[IMX6SX_CLK_EIM_SLOW_PODF] = imx_clk_hw_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3); + hws[IMX6SX_CLK_LCDIF2_PODF] = imx_clk_hw_divider("lcdif2_podf", "lcdif2_pred", base + 0x1c, 20, 3); + hws[IMX6SX_CLK_PERCLK] = imx_clk_hw_divider_flags("perclk", "perclk_sel", base + 0x1c, 0, 6, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_VID_PODF] = imx_clk_hw_divider("vid_podf", "vid_sel", base + 0x20, 24, 2); + hws[IMX6SX_CLK_CAN_PODF] = imx_clk_hw_divider("can_podf", "can_sel", base + 0x20, 2, 6); + hws[IMX6SX_CLK_USDHC4_PODF] = imx_clk_hw_divider("usdhc4_podf", "usdhc4_sel", base + 0x24, 22, 3); + hws[IMX6SX_CLK_USDHC3_PODF] = imx_clk_hw_divider("usdhc3_podf", "usdhc3_sel", base + 0x24, 19, 3); + hws[IMX6SX_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); + hws[IMX6SX_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); + hws[IMX6SX_CLK_UART_PODF] = imx_clk_hw_divider("uart_podf", "uart_sel", base + 0x24, 0, 6); + hws[IMX6SX_CLK_ESAI_PRED] = imx_clk_hw_divider("esai_pred", "esai_sel", base + 0x28, 9, 3); + hws[IMX6SX_CLK_ESAI_PODF] = imx_clk_hw_divider("esai_podf", "esai_pred", base + 0x28, 25, 3); + hws[IMX6SX_CLK_SSI3_PRED] = imx_clk_hw_divider("ssi3_pred", "ssi3_sel", base + 0x28, 22, 3); + hws[IMX6SX_CLK_SSI3_PODF] = imx_clk_hw_divider("ssi3_podf", "ssi3_pred", base + 0x28, 16, 6); + hws[IMX6SX_CLK_SSI1_PRED] = imx_clk_hw_divider("ssi1_pred", "ssi1_sel", base + 0x28, 6, 3); + hws[IMX6SX_CLK_SSI1_PODF] = imx_clk_hw_divider("ssi1_podf", "ssi1_pred", base + 0x28, 0, 6); + hws[IMX6SX_CLK_QSPI2_PRED] = imx_clk_hw_divider("qspi2_pred", "qspi2_sel", base + 0x2c, 18, 3); + hws[IMX6SX_CLK_QSPI2_PODF] = imx_clk_hw_divider("qspi2_podf", "qspi2_pred", base + 0x2c, 21, 6); + hws[IMX6SX_CLK_SSI2_PRED] = imx_clk_hw_divider("ssi2_pred", "ssi2_sel", base + 0x2c, 6, 3); + hws[IMX6SX_CLK_SSI2_PODF] = imx_clk_hw_divider("ssi2_podf", "ssi2_pred", base + 0x2c, 0, 6); + hws[IMX6SX_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); + hws[IMX6SX_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); + hws[IMX6SX_CLK_AUDIO_PRED] = imx_clk_hw_divider("audio_pred", "audio_sel", base + 0x30, 12, 3); + hws[IMX6SX_CLK_AUDIO_PODF] = imx_clk_hw_divider("audio_podf", "audio_pred", base + 0x30, 9, 3); + hws[IMX6SX_CLK_ENET_PODF] = imx_clk_hw_divider("enet_podf", "enet_pre_sel", base + 0x34, 12, 3); + hws[IMX6SX_CLK_M4_PODF] = imx_clk_hw_divider("m4_podf", "m4_sel", base + 0x34, 3, 3); + hws[IMX6SX_CLK_ECSPI_PODF] = imx_clk_hw_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6); + hws[IMX6SX_CLK_LCDIF1_PRED] = imx_clk_hw_divider("lcdif1_pred", "lcdif1_pre_sel", base + 0x38, 12, 3); + hws[IMX6SX_CLK_LCDIF2_PRED] = imx_clk_hw_divider("lcdif2_pred", "lcdif2_pre_sel", base + 0x38, 3, 3); + hws[IMX6SX_CLK_DISPLAY_PODF] = imx_clk_hw_divider("display_podf", "display_sel", base + 0x3c, 16, 3); + hws[IMX6SX_CLK_CSI_PODF] = imx_clk_hw_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3); + hws[IMX6SX_CLK_CKO1_PODF] = imx_clk_hw_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3); + hws[IMX6SX_CLK_CKO2_PODF] = imx_clk_hw_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3); + + hws[IMX6SX_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); + hws[IMX6SX_CLK_LDB_DI0_DIV_7] = imx_clk_hw_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7); + hws[IMX6SX_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7); + hws[IMX6SX_CLK_LDB_DI1_DIV_7] = imx_clk_hw_fixed_factor("ldb_di1_div_7", "ldb_di1_sel", 1, 7); /* name reg shift width busy: reg, shift parent_names num_parents */ - clks[IMX6SX_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); - clks[IMX6SX_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); + hws[IMX6SX_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); + hws[IMX6SX_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); /* name parent_name reg shift width busy: reg, shift */ - clks[IMX6SX_CLK_OCRAM_PODF] = imx_clk_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0); - clks[IMX6SX_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); - clks[IMX6SX_CLK_MMDC_PODF] = imx_clk_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); - clks[IMX6SX_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); + hws[IMX6SX_CLK_OCRAM_PODF] = imx_clk_hw_busy_divider("ocram_podf", "ocram_sel", base + 0x14, 16, 3, base + 0x48, 0); + hws[IMX6SX_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); + hws[IMX6SX_CLK_MMDC_PODF] = imx_clk_hw_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); + hws[IMX6SX_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); /* name parent_name reg shift */ /* CCGR0 */ - clks[IMX6SX_CLK_AIPS_TZ1] = imx_clk_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_AIPS_TZ2] = imx_clk_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_APBH_DMA] = imx_clk_gate2("apbh_dma", "usdhc3", base + 0x68, 4); - clks[IMX6SX_CLK_ASRC_MEM] = imx_clk_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc); - clks[IMX6SX_CLK_ASRC_IPG] = imx_clk_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc); - clks[IMX6SX_CLK_CAAM_MEM] = imx_clk_gate2("caam_mem", "ahb", base + 0x68, 8); - clks[IMX6SX_CLK_CAAM_ACLK] = imx_clk_gate2("caam_aclk", "ahb", base + 0x68, 10); - clks[IMX6SX_CLK_CAAM_IPG] = imx_clk_gate2("caam_ipg", "ipg", base + 0x68, 12); - clks[IMX6SX_CLK_CAN1_IPG] = imx_clk_gate2("can1_ipg", "ipg", base + 0x68, 14); - clks[IMX6SX_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_podf", base + 0x68, 16); - clks[IMX6SX_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18); - clks[IMX6SX_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_podf", base + 0x68, 20); - clks[IMX6SX_CLK_DCIC1] = imx_clk_gate2("dcic1", "display_podf", base + 0x68, 24); - clks[IMX6SX_CLK_DCIC2] = imx_clk_gate2("dcic2", "display_podf", base + 0x68, 26); - clks[IMX6SX_CLK_AIPS_TZ3] = imx_clk_gate2_flags("aips_tz3", "ahb", base + 0x68, 30, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_AIPS_TZ1] = imx_clk_hw_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_AIPS_TZ2] = imx_clk_hw_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_APBH_DMA] = imx_clk_hw_gate2("apbh_dma", "usdhc3", base + 0x68, 4); + hws[IMX6SX_CLK_ASRC_MEM] = imx_clk_hw_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc); + hws[IMX6SX_CLK_ASRC_IPG] = imx_clk_hw_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc); + hws[IMX6SX_CLK_CAAM_MEM] = imx_clk_hw_gate2("caam_mem", "ahb", base + 0x68, 8); + hws[IMX6SX_CLK_CAAM_ACLK] = imx_clk_hw_gate2("caam_aclk", "ahb", base + 0x68, 10); + hws[IMX6SX_CLK_CAAM_IPG] = imx_clk_hw_gate2("caam_ipg", "ipg", base + 0x68, 12); + hws[IMX6SX_CLK_CAN1_IPG] = imx_clk_hw_gate2("can1_ipg", "ipg", base + 0x68, 14); + hws[IMX6SX_CLK_CAN1_SERIAL] = imx_clk_hw_gate2("can1_serial", "can_podf", base + 0x68, 16); + hws[IMX6SX_CLK_CAN2_IPG] = imx_clk_hw_gate2("can2_ipg", "ipg", base + 0x68, 18); + hws[IMX6SX_CLK_CAN2_SERIAL] = imx_clk_hw_gate2("can2_serial", "can_podf", base + 0x68, 20); + hws[IMX6SX_CLK_DCIC1] = imx_clk_hw_gate2("dcic1", "display_podf", base + 0x68, 24); + hws[IMX6SX_CLK_DCIC2] = imx_clk_hw_gate2("dcic2", "display_podf", base + 0x68, 26); + hws[IMX6SX_CLK_AIPS_TZ3] = imx_clk_hw_gate2_flags("aips_tz3", "ahb", base + 0x68, 30, CLK_IS_CRITICAL); /* CCGR1 */ - clks[IMX6SX_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0); - clks[IMX6SX_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2); - clks[IMX6SX_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4); - clks[IMX6SX_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6); - clks[IMX6SX_CLK_ECSPI5] = imx_clk_gate2("ecspi5", "ecspi_podf", base + 0x6c, 8); - clks[IMX6SX_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12); - clks[IMX6SX_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14); - clks[IMX6SX_CLK_ESAI_EXTAL] = imx_clk_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai); - clks[IMX6SX_CLK_ESAI_IPG] = imx_clk_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai); - clks[IMX6SX_CLK_ESAI_MEM] = imx_clk_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai); - clks[IMX6SX_CLK_WAKEUP] = imx_clk_gate2_flags("wakeup", "ipg", base + 0x6c, 18, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_GPT_BUS] = imx_clk_gate2("gpt_bus", "perclk", base + 0x6c, 20); - clks[IMX6SX_CLK_GPT_SERIAL] = imx_clk_gate2("gpt_serial", "perclk", base + 0x6c, 22); - clks[IMX6SX_CLK_GPU] = imx_clk_gate2("gpu", "gpu_core_podf", base + 0x6c, 26); - clks[IMX6SX_CLK_OCRAM_S] = imx_clk_gate2("ocram_s", "ahb", base + 0x6c, 28); - clks[IMX6SX_CLK_CANFD] = imx_clk_gate2("canfd", "can_podf", base + 0x6c, 30); + hws[IMX6SX_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0); + hws[IMX6SX_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2); + hws[IMX6SX_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4); + hws[IMX6SX_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6); + hws[IMX6SX_CLK_ECSPI5] = imx_clk_hw_gate2("ecspi5", "ecspi_podf", base + 0x6c, 8); + hws[IMX6SX_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12); + hws[IMX6SX_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14); + hws[IMX6SX_CLK_ESAI_EXTAL] = imx_clk_hw_gate2_shared("esai_extal", "esai_podf", base + 0x6c, 16, &share_count_esai); + hws[IMX6SX_CLK_ESAI_IPG] = imx_clk_hw_gate2_shared("esai_ipg", "ahb", base + 0x6c, 16, &share_count_esai); + hws[IMX6SX_CLK_ESAI_MEM] = imx_clk_hw_gate2_shared("esai_mem", "ahb", base + 0x6c, 16, &share_count_esai); + hws[IMX6SX_CLK_WAKEUP] = imx_clk_hw_gate2_flags("wakeup", "ipg", base + 0x6c, 18, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_GPT_BUS] = imx_clk_hw_gate2("gpt_bus", "perclk", base + 0x6c, 20); + hws[IMX6SX_CLK_GPT_SERIAL] = imx_clk_hw_gate2("gpt_serial", "perclk", base + 0x6c, 22); + hws[IMX6SX_CLK_GPU] = imx_clk_hw_gate2("gpu", "gpu_core_podf", base + 0x6c, 26); + hws[IMX6SX_CLK_OCRAM_S] = imx_clk_hw_gate2("ocram_s", "ahb", base + 0x6c, 28); + hws[IMX6SX_CLK_CANFD] = imx_clk_hw_gate2("canfd", "can_podf", base + 0x6c, 30); /* CCGR2 */ - clks[IMX6SX_CLK_CSI] = imx_clk_gate2("csi", "csi_podf", base + 0x70, 2); - clks[IMX6SX_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6); - clks[IMX6SX_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8); - clks[IMX6SX_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10); - clks[IMX6SX_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12); - clks[IMX6SX_CLK_IOMUXC] = imx_clk_gate2("iomuxc", "lcdif1_podf", base + 0x70, 14); - clks[IMX6SX_CLK_IPMUX1] = imx_clk_gate2_flags("ipmux1", "ahb", base + 0x70, 16, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_IPMUX2] = imx_clk_gate2_flags("ipmux2", "ahb", base + 0x70, 18, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_IPMUX3] = imx_clk_gate2_flags("ipmux3", "ahb", base + 0x70, 20, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_TZASC1] = imx_clk_gate2_flags("tzasc1", "mmdc_podf", base + 0x70, 22, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_LCDIF_APB] = imx_clk_gate2("lcdif_apb", "display_podf", base + 0x70, 28); - clks[IMX6SX_CLK_PXP_AXI] = imx_clk_gate2("pxp_axi", "display_podf", base + 0x70, 30); + hws[IMX6SX_CLK_CSI] = imx_clk_hw_gate2("csi", "csi_podf", base + 0x70, 2); + hws[IMX6SX_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6); + hws[IMX6SX_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8); + hws[IMX6SX_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10); + hws[IMX6SX_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12); + hws[IMX6SX_CLK_IOMUXC] = imx_clk_hw_gate2("iomuxc", "lcdif1_podf", base + 0x70, 14); + hws[IMX6SX_CLK_IPMUX1] = imx_clk_hw_gate2_flags("ipmux1", "ahb", base + 0x70, 16, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_IPMUX2] = imx_clk_hw_gate2_flags("ipmux2", "ahb", base + 0x70, 18, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_IPMUX3] = imx_clk_hw_gate2_flags("ipmux3", "ahb", base + 0x70, 20, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_TZASC1] = imx_clk_hw_gate2_flags("tzasc1", "mmdc_podf", base + 0x70, 22, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_LCDIF_APB] = imx_clk_hw_gate2("lcdif_apb", "display_podf", base + 0x70, 28); + hws[IMX6SX_CLK_PXP_AXI] = imx_clk_hw_gate2("pxp_axi", "display_podf", base + 0x70, 30); /* CCGR3 */ - clks[IMX6SX_CLK_M4] = imx_clk_gate2("m4", "m4_podf", base + 0x74, 2); - clks[IMX6SX_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x74, 4); - clks[IMX6SX_CLK_ENET_AHB] = imx_clk_gate2("enet_ahb", "enet_sel", base + 0x74, 4); - clks[IMX6SX_CLK_DISPLAY_AXI] = imx_clk_gate2("display_axi", "display_podf", base + 0x74, 6); - clks[IMX6SX_CLK_LCDIF2_PIX] = imx_clk_gate2("lcdif2_pix", "lcdif2_sel", base + 0x74, 8); - clks[IMX6SX_CLK_LCDIF1_PIX] = imx_clk_gate2("lcdif1_pix", "lcdif1_sel", base + 0x74, 10); - clks[IMX6SX_CLK_LDB_DI0] = imx_clk_gate2("ldb_di0", "ldb_di0_div_sel", base + 0x74, 12); - clks[IMX6SX_CLK_QSPI1] = imx_clk_gate2("qspi1", "qspi1_podf", base + 0x74, 14); - clks[IMX6SX_CLK_MLB] = imx_clk_gate2("mlb", "ahb", base + 0x74, 18); - clks[IMX6SX_CLK_MMDC_P0_FAST] = imx_clk_gate2_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26); - clks[IMX6SX_CLK_OCRAM] = imx_clk_gate2_flags("ocram", "ocram_podf", base + 0x74, 28, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_M4] = imx_clk_hw_gate2("m4", "m4_podf", base + 0x74, 2); + hws[IMX6SX_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x74, 4); + hws[IMX6SX_CLK_ENET_AHB] = imx_clk_hw_gate2("enet_ahb", "enet_sel", base + 0x74, 4); + hws[IMX6SX_CLK_DISPLAY_AXI] = imx_clk_hw_gate2("display_axi", "display_podf", base + 0x74, 6); + hws[IMX6SX_CLK_LCDIF2_PIX] = imx_clk_hw_gate2("lcdif2_pix", "lcdif2_sel", base + 0x74, 8); + hws[IMX6SX_CLK_LCDIF1_PIX] = imx_clk_hw_gate2("lcdif1_pix", "lcdif1_sel", base + 0x74, 10); + hws[IMX6SX_CLK_LDB_DI0] = imx_clk_hw_gate2("ldb_di0", "ldb_di0_div_sel", base + 0x74, 12); + hws[IMX6SX_CLK_QSPI1] = imx_clk_hw_gate2("qspi1", "qspi1_podf", base + 0x74, 14); + hws[IMX6SX_CLK_MLB] = imx_clk_hw_gate2("mlb", "ahb", base + 0x74, 18); + hws[IMX6SX_CLK_MMDC_P0_FAST] = imx_clk_hw_gate2_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2_flags("mmdc_p1_ipg", "ipg", base + 0x74, 26, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_OCRAM] = imx_clk_hw_gate2_flags("ocram", "ocram_podf", base + 0x74, 28, CLK_IS_CRITICAL); /* CCGR4 */ - clks[IMX6SX_CLK_PCIE_AXI] = imx_clk_gate2("pcie_axi", "display_podf", base + 0x78, 0); - clks[IMX6SX_CLK_QSPI2] = imx_clk_gate2("qspi2", "qspi2_podf", base + 0x78, 10); - clks[IMX6SX_CLK_PER1_BCH] = imx_clk_gate2("per1_bch", "usdhc3", base + 0x78, 12); - clks[IMX6SX_CLK_PER2_MAIN] = imx_clk_gate2_flags("per2_main", "ahb", base + 0x78, 14, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16); - clks[IMX6SX_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18); - clks[IMX6SX_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20); - clks[IMX6SX_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22); - clks[IMX6SX_CLK_GPMI_BCH_APB] = imx_clk_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24); - clks[IMX6SX_CLK_GPMI_BCH] = imx_clk_gate2("gpmi_bch", "usdhc4", base + 0x78, 26); - clks[IMX6SX_CLK_GPMI_IO] = imx_clk_gate2("gpmi_io", "qspi2_podf", base + 0x78, 28); - clks[IMX6SX_CLK_GPMI_APB] = imx_clk_gate2("gpmi_apb", "usdhc3", base + 0x78, 30); + hws[IMX6SX_CLK_PCIE_AXI] = imx_clk_hw_gate2("pcie_axi", "display_podf", base + 0x78, 0); + hws[IMX6SX_CLK_QSPI2] = imx_clk_hw_gate2("qspi2", "qspi2_podf", base + 0x78, 10); + hws[IMX6SX_CLK_PER1_BCH] = imx_clk_hw_gate2("per1_bch", "usdhc3", base + 0x78, 12); + hws[IMX6SX_CLK_PER2_MAIN] = imx_clk_hw_gate2_flags("per2_main", "ahb", base + 0x78, 14, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16); + hws[IMX6SX_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18); + hws[IMX6SX_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20); + hws[IMX6SX_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22); + hws[IMX6SX_CLK_GPMI_BCH_APB] = imx_clk_hw_gate2("gpmi_bch_apb", "usdhc3", base + 0x78, 24); + hws[IMX6SX_CLK_GPMI_BCH] = imx_clk_hw_gate2("gpmi_bch", "usdhc4", base + 0x78, 26); + hws[IMX6SX_CLK_GPMI_IO] = imx_clk_hw_gate2("gpmi_io", "qspi2_podf", base + 0x78, 28); + hws[IMX6SX_CLK_GPMI_APB] = imx_clk_hw_gate2("gpmi_apb", "usdhc3", base + 0x78, 30); /* CCGR5 */ - clks[IMX6SX_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); - clks[IMX6SX_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6); - clks[IMX6SX_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12); - clks[IMX6SX_CLK_AUDIO] = imx_clk_gate2_shared("audio", "audio_podf", base + 0x7c, 14, &share_count_audio); - clks[IMX6SX_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio); - clks[IMX6SX_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio); - clks[IMX6SX_CLK_SSI1_IPG] = imx_clk_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); - clks[IMX6SX_CLK_SSI2_IPG] = imx_clk_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); - clks[IMX6SX_CLK_SSI3_IPG] = imx_clk_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); - clks[IMX6SX_CLK_SSI1] = imx_clk_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); - clks[IMX6SX_CLK_SSI2] = imx_clk_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); - clks[IMX6SX_CLK_SSI3] = imx_clk_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); - clks[IMX6SX_CLK_UART_IPG] = imx_clk_gate2("uart_ipg", "ipg", base + 0x7c, 24); - clks[IMX6SX_CLK_UART_SERIAL] = imx_clk_gate2("uart_serial", "uart_podf", base + 0x7c, 26); - clks[IMX6SX_CLK_SAI1_IPG] = imx_clk_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1); - clks[IMX6SX_CLK_SAI2_IPG] = imx_clk_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2); - clks[IMX6SX_CLK_SAI1] = imx_clk_gate2_shared("sai1", "ssi1_podf", base + 0x7c, 28, &share_count_sai1); - clks[IMX6SX_CLK_SAI2] = imx_clk_gate2_shared("sai2", "ssi2_podf", base + 0x7c, 30, &share_count_sai2); + hws[IMX6SX_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); + hws[IMX6SX_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6); + hws[IMX6SX_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12); + hws[IMX6SX_CLK_AUDIO] = imx_clk_hw_gate2_shared("audio", "audio_podf", base + 0x7c, 14, &share_count_audio); + hws[IMX6SX_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio); + hws[IMX6SX_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio); + hws[IMX6SX_CLK_SSI1_IPG] = imx_clk_hw_gate2_shared("ssi1_ipg", "ipg", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6SX_CLK_SSI2_IPG] = imx_clk_hw_gate2_shared("ssi2_ipg", "ipg", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6SX_CLK_SSI3_IPG] = imx_clk_hw_gate2_shared("ssi3_ipg", "ipg", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6SX_CLK_SSI1] = imx_clk_hw_gate2_shared("ssi1", "ssi1_podf", base + 0x7c, 18, &share_count_ssi1); + hws[IMX6SX_CLK_SSI2] = imx_clk_hw_gate2_shared("ssi2", "ssi2_podf", base + 0x7c, 20, &share_count_ssi2); + hws[IMX6SX_CLK_SSI3] = imx_clk_hw_gate2_shared("ssi3", "ssi3_podf", base + 0x7c, 22, &share_count_ssi3); + hws[IMX6SX_CLK_UART_IPG] = imx_clk_hw_gate2("uart_ipg", "ipg", base + 0x7c, 24); + hws[IMX6SX_CLK_UART_SERIAL] = imx_clk_hw_gate2("uart_serial", "uart_podf", base + 0x7c, 26); + hws[IMX6SX_CLK_SAI1_IPG] = imx_clk_hw_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1); + hws[IMX6SX_CLK_SAI2_IPG] = imx_clk_hw_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2); + hws[IMX6SX_CLK_SAI1] = imx_clk_hw_gate2_shared("sai1", "ssi1_podf", base + 0x7c, 28, &share_count_sai1); + hws[IMX6SX_CLK_SAI2] = imx_clk_hw_gate2_shared("sai2", "ssi2_podf", base + 0x7c, 30, &share_count_sai2); /* CCGR6 */ - clks[IMX6SX_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0); - clks[IMX6SX_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); - clks[IMX6SX_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); - clks[IMX6SX_CLK_USDHC3] = imx_clk_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); - clks[IMX6SX_CLK_USDHC4] = imx_clk_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8); - clks[IMX6SX_CLK_EIM_SLOW] = imx_clk_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10); - clks[IMX6SX_CLK_PWM8] = imx_clk_gate2("pwm8", "perclk", base + 0x80, 16); - clks[IMX6SX_CLK_VADC] = imx_clk_gate2("vadc", "vid_podf", base + 0x80, 20); - clks[IMX6SX_CLK_GIS] = imx_clk_gate2("gis", "display_podf", base + 0x80, 22); - clks[IMX6SX_CLK_I2C4] = imx_clk_gate2("i2c4", "perclk", base + 0x80, 24); - clks[IMX6SX_CLK_PWM5] = imx_clk_gate2("pwm5", "perclk", base + 0x80, 26); - clks[IMX6SX_CLK_PWM6] = imx_clk_gate2("pwm6", "perclk", base + 0x80, 28); - clks[IMX6SX_CLK_PWM7] = imx_clk_gate2("pwm7", "perclk", base + 0x80, 30); - - clks[IMX6SX_CLK_CKO1] = imx_clk_gate("cko1", "cko1_podf", base + 0x60, 7); - clks[IMX6SX_CLK_CKO2] = imx_clk_gate("cko2", "cko2_podf", base + 0x60, 24); + hws[IMX6SX_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0); + hws[IMX6SX_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); + hws[IMX6SX_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); + hws[IMX6SX_CLK_USDHC3] = imx_clk_hw_gate2("usdhc3", "usdhc3_podf", base + 0x80, 6); + hws[IMX6SX_CLK_USDHC4] = imx_clk_hw_gate2("usdhc4", "usdhc4_podf", base + 0x80, 8); + hws[IMX6SX_CLK_EIM_SLOW] = imx_clk_hw_gate2("eim_slow", "eim_slow_podf", base + 0x80, 10); + hws[IMX6SX_CLK_PWM8] = imx_clk_hw_gate2("pwm8", "perclk", base + 0x80, 16); + hws[IMX6SX_CLK_VADC] = imx_clk_hw_gate2("vadc", "vid_podf", base + 0x80, 20); + hws[IMX6SX_CLK_GIS] = imx_clk_hw_gate2("gis", "display_podf", base + 0x80, 22); + hws[IMX6SX_CLK_I2C4] = imx_clk_hw_gate2("i2c4", "perclk", base + 0x80, 24); + hws[IMX6SX_CLK_PWM5] = imx_clk_hw_gate2("pwm5", "perclk", base + 0x80, 26); + hws[IMX6SX_CLK_PWM6] = imx_clk_hw_gate2("pwm6", "perclk", base + 0x80, 28); + hws[IMX6SX_CLK_PWM7] = imx_clk_hw_gate2("pwm7", "perclk", base + 0x80, 30); + + hws[IMX6SX_CLK_CKO1] = imx_clk_hw_gate("cko1", "cko1_podf", base + 0x60, 7); + hws[IMX6SX_CLK_CKO2] = imx_clk_hw_gate("cko2", "cko2_podf", base + 0x60, 24); /* mask handshake of mmdc */ - writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR); + imx_mmdc_mask_handshake(base, 0); - imx_check_clocks(clks, ARRAY_SIZE(clks)); + imx_check_clk_hws(hws, IMX6SX_CLK_CLK_END); - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); if (IS_ENABLED(CONFIG_USB_MXS_PHY)) { - clk_prepare_enable(clks[IMX6SX_CLK_USBPHY1_GATE]); - clk_prepare_enable(clks[IMX6SX_CLK_USBPHY2_GATE]); + clk_prepare_enable(hws[IMX6SX_CLK_USBPHY1_GATE]->clk); + clk_prepare_enable(hws[IMX6SX_CLK_USBPHY2_GATE]->clk); } /* Set the default 132MHz for EIM module */ - clk_set_parent(clks[IMX6SX_CLK_EIM_SLOW_SEL], clks[IMX6SX_CLK_PLL2_PFD2]); - clk_set_rate(clks[IMX6SX_CLK_EIM_SLOW], 132000000); + clk_set_parent(hws[IMX6SX_CLK_EIM_SLOW_SEL]->clk, hws[IMX6SX_CLK_PLL2_PFD2]->clk); + clk_set_rate(hws[IMX6SX_CLK_EIM_SLOW]->clk, 132000000); /* set parent clock for LCDIF1 pixel clock */ - clk_set_parent(clks[IMX6SX_CLK_LCDIF1_PRE_SEL], clks[IMX6SX_CLK_PLL5_VIDEO_DIV]); - clk_set_parent(clks[IMX6SX_CLK_LCDIF1_SEL], clks[IMX6SX_CLK_LCDIF1_PODF]); + clk_set_parent(hws[IMX6SX_CLK_LCDIF1_PRE_SEL]->clk, hws[IMX6SX_CLK_PLL5_VIDEO_DIV]->clk); + clk_set_parent(hws[IMX6SX_CLK_LCDIF1_SEL]->clk, hws[IMX6SX_CLK_LCDIF1_PODF]->clk); /* Set the parent clks of PCIe lvds1 and pcie_axi to be pcie ref, axi */ - if (clk_set_parent(clks[IMX6SX_CLK_LVDS1_SEL], clks[IMX6SX_CLK_PCIE_REF_125M])) + if (clk_set_parent(hws[IMX6SX_CLK_LVDS1_SEL]->clk, hws[IMX6SX_CLK_PCIE_REF_125M]->clk)) pr_err("Failed to set pcie bus parent clk.\n"); - if (clk_set_parent(clks[IMX6SX_CLK_PCIE_AXI_SEL], clks[IMX6SX_CLK_AXI])) - pr_err("Failed to set pcie parent clk.\n"); /* * Init enet system AHB clock, set to 200MHz * pll2_pfd2_396m-> ENET_PODF-> ENET_AHB */ - clk_set_parent(clks[IMX6SX_CLK_ENET_PRE_SEL], clks[IMX6SX_CLK_PLL2_PFD2]); - clk_set_parent(clks[IMX6SX_CLK_ENET_SEL], clks[IMX6SX_CLK_ENET_PODF]); - clk_set_rate(clks[IMX6SX_CLK_ENET_PODF], 200000000); - clk_set_rate(clks[IMX6SX_CLK_ENET_REF], 125000000); - clk_set_rate(clks[IMX6SX_CLK_ENET2_REF], 125000000); + clk_set_parent(hws[IMX6SX_CLK_ENET_PRE_SEL]->clk, hws[IMX6SX_CLK_PLL2_PFD2]->clk); + clk_set_parent(hws[IMX6SX_CLK_ENET_SEL]->clk, hws[IMX6SX_CLK_ENET_PODF]->clk); + clk_set_rate(hws[IMX6SX_CLK_ENET_PODF]->clk, 200000000); + clk_set_rate(hws[IMX6SX_CLK_ENET_REF]->clk, 125000000); + clk_set_rate(hws[IMX6SX_CLK_ENET2_REF]->clk, 125000000); /* Audio clocks */ - clk_set_rate(clks[IMX6SX_CLK_PLL4_AUDIO_DIV], 393216000); + clk_set_rate(hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk, 393216000); - clk_set_parent(clks[IMX6SX_CLK_SPDIF_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]); - clk_set_rate(clks[IMX6SX_CLK_SPDIF_PODF], 98304000); + clk_set_parent(hws[IMX6SX_CLK_SPDIF_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk); + clk_set_rate(hws[IMX6SX_CLK_SPDIF_PODF]->clk, 98304000); - clk_set_parent(clks[IMX6SX_CLK_AUDIO_SEL], clks[IMX6SX_CLK_PLL3_USB_OTG]); - clk_set_rate(clks[IMX6SX_CLK_AUDIO_PODF], 24000000); + clk_set_parent(hws[IMX6SX_CLK_AUDIO_SEL]->clk, hws[IMX6SX_CLK_PLL3_USB_OTG]->clk); + clk_set_rate(hws[IMX6SX_CLK_AUDIO_PODF]->clk, 24000000); - clk_set_parent(clks[IMX6SX_CLK_SSI1_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]); - clk_set_parent(clks[IMX6SX_CLK_SSI2_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]); - clk_set_parent(clks[IMX6SX_CLK_SSI3_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]); - clk_set_rate(clks[IMX6SX_CLK_SSI1_PODF], 24576000); - clk_set_rate(clks[IMX6SX_CLK_SSI2_PODF], 24576000); - clk_set_rate(clks[IMX6SX_CLK_SSI3_PODF], 24576000); + clk_set_parent(hws[IMX6SX_CLK_SSI1_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk); + clk_set_parent(hws[IMX6SX_CLK_SSI2_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk); + clk_set_parent(hws[IMX6SX_CLK_SSI3_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk); + clk_set_rate(hws[IMX6SX_CLK_SSI1_PODF]->clk, 24576000); + clk_set_rate(hws[IMX6SX_CLK_SSI2_PODF]->clk, 24576000); + clk_set_rate(hws[IMX6SX_CLK_SSI3_PODF]->clk, 24576000); - clk_set_parent(clks[IMX6SX_CLK_ESAI_SEL], clks[IMX6SX_CLK_PLL4_AUDIO_DIV]); - clk_set_rate(clks[IMX6SX_CLK_ESAI_PODF], 24576000); + clk_set_parent(hws[IMX6SX_CLK_ESAI_SEL]->clk, hws[IMX6SX_CLK_PLL4_AUDIO_DIV]->clk); + clk_set_rate(hws[IMX6SX_CLK_ESAI_PODF]->clk, 24576000); /* Set parent clock for vadc */ - clk_set_parent(clks[IMX6SX_CLK_VID_SEL], clks[IMX6SX_CLK_PLL3_USB_OTG]); + clk_set_parent(hws[IMX6SX_CLK_VID_SEL]->clk, hws[IMX6SX_CLK_PLL3_USB_OTG]->clk); /* default parent of can_sel clock is invalid, manually set it here */ - clk_set_parent(clks[IMX6SX_CLK_CAN_SEL], clks[IMX6SX_CLK_PLL3_60M]); + clk_set_parent(hws[IMX6SX_CLK_CAN_SEL]->clk, hws[IMX6SX_CLK_PLL3_60M]->clk); /* Update gpu clock from default 528M to 720M */ - clk_set_parent(clks[IMX6SX_CLK_GPU_CORE_SEL], clks[IMX6SX_CLK_PLL3_PFD0]); - clk_set_parent(clks[IMX6SX_CLK_GPU_AXI_SEL], clks[IMX6SX_CLK_PLL3_PFD0]); + clk_set_parent(hws[IMX6SX_CLK_GPU_CORE_SEL]->clk, hws[IMX6SX_CLK_PLL3_PFD0]->clk); + clk_set_parent(hws[IMX6SX_CLK_GPU_AXI_SEL]->clk, hws[IMX6SX_CLK_PLL3_PFD0]->clk); - clk_set_parent(clks[IMX6SX_CLK_QSPI1_SEL], clks[IMX6SX_CLK_PLL2_BUS]); - clk_set_parent(clks[IMX6SX_CLK_QSPI2_SEL], clks[IMX6SX_CLK_PLL2_BUS]); + clk_set_parent(hws[IMX6SX_CLK_QSPI1_SEL]->clk, hws[IMX6SX_CLK_PLL2_BUS]->clk); + clk_set_parent(hws[IMX6SX_CLK_QSPI2_SEL]->clk, hws[IMX6SX_CLK_PLL2_BUS]->clk); + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_clks[i] = &hws[index]->clk; + } imx_register_uart_clocks(uart_clks); } diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c index 8fd52e103cc2..bc931988fe7b 100644 --- a/drivers/clk/imx/clk-imx6ul.c +++ b/drivers/clk/imx/clk-imx6ul.c @@ -6,6 +6,7 @@ #include <dt-bindings/clock/imx6ul-clock.h> #include <linux/clk.h> #include <linux/clkdev.h> +#include <linux/clk-provider.h> #include <linux/err.h> #include <linux/init.h> #include <linux/io.h> @@ -16,9 +17,6 @@ #include "clk.h" -#define BM_CCM_CCDR_MMDC_CH0_MASK (0x2 << 16) -#define CCDR 0x4 - static const char *pll_bypass_src_sels[] = { "osc", "dummy", }; static const char *pll1_bypass_sels[] = { "pll1", "pll1_bypass_src", }; static const char *pll2_bypass_sels[] = { "pll2", "pll2_bypass_src", }; @@ -70,8 +68,8 @@ static const char *cko2_sels[] = { "dummy", "dummy", "dummy", "usdhc1", "dummy", "dummy", "dummy", "dummy", "dummy", "uart_serial", "spdif", "dummy", "dummy", }; static const char *cko_sels[] = { "cko1", "cko2", }; -static struct clk *clks[IMX6UL_CLK_END]; -static struct clk_onecell_data clk_data; +static struct clk_hw **hws; +static struct clk_hw_onecell_data *clk_hw_data; static const struct clk_div_table clk_enet_ref_table[] = { { .val = 0, .div = 20, }, @@ -118,61 +116,69 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node) struct device_node *np; void __iomem *base; - clks[IMX6UL_CLK_DUMMY] = imx_clk_fixed("dummy", 0); + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX6UL_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return; + + clk_hw_data->num = IMX6UL_CLK_END; + hws = clk_hw_data->hws; + + hws[IMX6UL_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); - clks[IMX6UL_CLK_CKIL] = of_clk_get_by_name(ccm_node, "ckil"); - clks[IMX6UL_CLK_OSC] = of_clk_get_by_name(ccm_node, "osc"); + hws[IMX6UL_CLK_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil")); + hws[IMX6UL_CLK_OSC] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc")); /* ipp_di clock is external input */ - clks[IMX6UL_CLK_IPP_DI0] = of_clk_get_by_name(ccm_node, "ipp_di0"); - clks[IMX6UL_CLK_IPP_DI1] = of_clk_get_by_name(ccm_node, "ipp_di1"); + hws[IMX6UL_CLK_IPP_DI0] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di0")); + hws[IMX6UL_CLK_IPP_DI1] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ipp_di1")); np = of_find_compatible_node(NULL, NULL, "fsl,imx6ul-anatop"); base = of_iomap(np, 0); of_node_put(np); WARN_ON(!base); - clks[IMX6UL_PLL1_BYPASS_SRC] = imx_clk_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6UL_PLL2_BYPASS_SRC] = imx_clk_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6UL_PLL3_BYPASS_SRC] = imx_clk_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6UL_PLL4_BYPASS_SRC] = imx_clk_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6UL_PLL5_BYPASS_SRC] = imx_clk_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6UL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6UL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - - clks[IMX6UL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); - clks[IMX6UL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); - clks[IMX6UL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); - clks[IMX6UL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); - clks[IMX6UL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); - clks[IMX6UL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); - clks[IMX6UL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); - - clks[IMX6UL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_PLL3_BYPASS] = imx_clk_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_PLL4_BYPASS] = imx_clk_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_PLL5_BYPASS] = imx_clk_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_PLL6_BYPASS] = imx_clk_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_PLL7_BYPASS] = imx_clk_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_CLK_CSI_SEL] = imx_clk_mux_flags("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_PLL1_BYPASS_SRC] = imx_clk_hw_mux("pll1_bypass_src", base + 0x00, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6UL_PLL2_BYPASS_SRC] = imx_clk_hw_mux("pll2_bypass_src", base + 0x30, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6UL_PLL3_BYPASS_SRC] = imx_clk_hw_mux("pll3_bypass_src", base + 0x10, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6UL_PLL4_BYPASS_SRC] = imx_clk_hw_mux("pll4_bypass_src", base + 0x70, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6UL_PLL5_BYPASS_SRC] = imx_clk_hw_mux("pll5_bypass_src", base + 0xa0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6UL_PLL6_BYPASS_SRC] = imx_clk_hw_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + hws[IMX6UL_PLL7_BYPASS_SRC] = imx_clk_hw_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); + + hws[IMX6UL_CLK_PLL1] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + hws[IMX6UL_CLK_PLL2] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + hws[IMX6UL_CLK_PLL3] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + hws[IMX6UL_CLK_PLL4] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + hws[IMX6UL_CLK_PLL5] = imx_clk_hw_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + hws[IMX6UL_CLK_PLL6] = imx_clk_hw_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + hws[IMX6UL_CLK_PLL7] = imx_clk_hw_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); + + hws[IMX6UL_PLL1_BYPASS] = imx_clk_hw_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_PLL2_BYPASS] = imx_clk_hw_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_PLL3_BYPASS] = imx_clk_hw_mux_flags("pll3_bypass", base + 0x10, 16, 1, pll3_bypass_sels, ARRAY_SIZE(pll3_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_PLL4_BYPASS] = imx_clk_hw_mux_flags("pll4_bypass", base + 0x70, 16, 1, pll4_bypass_sels, ARRAY_SIZE(pll4_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_PLL5_BYPASS] = imx_clk_hw_mux_flags("pll5_bypass", base + 0xa0, 16, 1, pll5_bypass_sels, ARRAY_SIZE(pll5_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_PLL6_BYPASS] = imx_clk_hw_mux_flags("pll6_bypass", base + 0xe0, 16, 1, pll6_bypass_sels, ARRAY_SIZE(pll6_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_PLL7_BYPASS] = imx_clk_hw_mux_flags("pll7_bypass", base + 0x20, 16, 1, pll7_bypass_sels, ARRAY_SIZE(pll7_bypass_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_CLK_CSI_SEL] = imx_clk_hw_mux_flags("csi_sel", base + 0x3c, 9, 2, csi_sels, ARRAY_SIZE(csi_sels), CLK_SET_RATE_PARENT); /* Do not bypass PLLs initially */ - clk_set_parent(clks[IMX6UL_PLL1_BYPASS], clks[IMX6UL_CLK_PLL1]); - clk_set_parent(clks[IMX6UL_PLL2_BYPASS], clks[IMX6UL_CLK_PLL2]); - clk_set_parent(clks[IMX6UL_PLL3_BYPASS], clks[IMX6UL_CLK_PLL3]); - clk_set_parent(clks[IMX6UL_PLL4_BYPASS], clks[IMX6UL_CLK_PLL4]); - clk_set_parent(clks[IMX6UL_PLL5_BYPASS], clks[IMX6UL_CLK_PLL5]); - clk_set_parent(clks[IMX6UL_PLL6_BYPASS], clks[IMX6UL_CLK_PLL6]); - clk_set_parent(clks[IMX6UL_PLL7_BYPASS], clks[IMX6UL_CLK_PLL7]); - - clks[IMX6UL_CLK_PLL1_SYS] = imx_clk_fixed_factor("pll1_sys", "pll1_bypass", 1, 1); - clks[IMX6UL_CLK_PLL2_BUS] = imx_clk_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); - clks[IMX6UL_CLK_PLL3_USB_OTG] = imx_clk_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); - clks[IMX6UL_CLK_PLL4_AUDIO] = imx_clk_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); - clks[IMX6UL_CLK_PLL5_VIDEO] = imx_clk_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); - clks[IMX6UL_CLK_PLL6_ENET] = imx_clk_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); - clks[IMX6UL_CLK_PLL7_USB_HOST] = imx_clk_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); + clk_set_parent(hws[IMX6UL_PLL1_BYPASS]->clk, hws[IMX6UL_CLK_PLL1]->clk); + clk_set_parent(hws[IMX6UL_PLL2_BYPASS]->clk, hws[IMX6UL_CLK_PLL2]->clk); + clk_set_parent(hws[IMX6UL_PLL3_BYPASS]->clk, hws[IMX6UL_CLK_PLL3]->clk); + clk_set_parent(hws[IMX6UL_PLL4_BYPASS]->clk, hws[IMX6UL_CLK_PLL4]->clk); + clk_set_parent(hws[IMX6UL_PLL5_BYPASS]->clk, hws[IMX6UL_CLK_PLL5]->clk); + clk_set_parent(hws[IMX6UL_PLL6_BYPASS]->clk, hws[IMX6UL_CLK_PLL6]->clk); + clk_set_parent(hws[IMX6UL_PLL7_BYPASS]->clk, hws[IMX6UL_CLK_PLL7]->clk); + + hws[IMX6UL_CLK_PLL1_SYS] = imx_clk_hw_fixed_factor("pll1_sys", "pll1_bypass", 1, 1); + hws[IMX6UL_CLK_PLL2_BUS] = imx_clk_hw_gate("pll2_bus", "pll2_bypass", base + 0x30, 13); + hws[IMX6UL_CLK_PLL3_USB_OTG] = imx_clk_hw_gate("pll3_usb_otg", "pll3_bypass", base + 0x10, 13); + hws[IMX6UL_CLK_PLL4_AUDIO] = imx_clk_hw_gate("pll4_audio", "pll4_bypass", base + 0x70, 13); + hws[IMX6UL_CLK_PLL5_VIDEO] = imx_clk_hw_gate("pll5_video", "pll5_bypass", base + 0xa0, 13); + hws[IMX6UL_CLK_PLL6_ENET] = imx_clk_hw_gate("pll6_enet", "pll6_bypass", base + 0xe0, 13); + hws[IMX6UL_CLK_PLL7_USB_HOST] = imx_clk_hw_gate("pll7_usb_host", "pll7_bypass", base + 0x20, 13); /* * Bit 20 is the reserved and read-only bit, we do this only for: @@ -180,291 +186,289 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node) * - Keep refcount when do usbphy clk_enable/disable, in that case, * the clk framework many need to enable/disable usbphy's parent */ - clks[IMX6UL_CLK_USBPHY1] = imx_clk_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); - clks[IMX6UL_CLK_USBPHY2] = imx_clk_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); + hws[IMX6UL_CLK_USBPHY1] = imx_clk_hw_gate("usbphy1", "pll3_usb_otg", base + 0x10, 20); + hws[IMX6UL_CLK_USBPHY2] = imx_clk_hw_gate("usbphy2", "pll7_usb_host", base + 0x20, 20); /* * usbphy*_gate needs to be on after system boots up, and software * never needs to control it anymore. */ - clks[IMX6UL_CLK_USBPHY1_GATE] = imx_clk_gate("usbphy1_gate", "dummy", base + 0x10, 6); - clks[IMX6UL_CLK_USBPHY2_GATE] = imx_clk_gate("usbphy2_gate", "dummy", base + 0x20, 6); + hws[IMX6UL_CLK_USBPHY1_GATE] = imx_clk_hw_gate("usbphy1_gate", "dummy", base + 0x10, 6); + hws[IMX6UL_CLK_USBPHY2_GATE] = imx_clk_hw_gate("usbphy2_gate", "dummy", base + 0x20, 6); /* name parent_name reg idx */ - clks[IMX6UL_CLK_PLL2_PFD0] = imx_clk_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); - clks[IMX6UL_CLK_PLL2_PFD1] = imx_clk_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); - clks[IMX6UL_CLK_PLL2_PFD2] = imx_clk_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); - clks[IMX6UL_CLK_PLL2_PFD3] = imx_clk_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3); - clks[IMX6UL_CLK_PLL3_PFD0] = imx_clk_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); - clks[IMX6UL_CLK_PLL3_PFD1] = imx_clk_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); - clks[IMX6UL_CLK_PLL3_PFD2] = imx_clk_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); - clks[IMX6UL_CLK_PLL3_PFD3] = imx_clk_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); - - clks[IMX6UL_CLK_ENET_REF] = clk_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, + hws[IMX6UL_CLK_PLL2_PFD0] = imx_clk_hw_pfd("pll2_pfd0_352m", "pll2_bus", base + 0x100, 0); + hws[IMX6UL_CLK_PLL2_PFD1] = imx_clk_hw_pfd("pll2_pfd1_594m", "pll2_bus", base + 0x100, 1); + hws[IMX6UL_CLK_PLL2_PFD2] = imx_clk_hw_pfd("pll2_pfd2_396m", "pll2_bus", base + 0x100, 2); + hws[IMX6UL_CLK_PLL2_PFD3] = imx_clk_hw_pfd("pll2_pfd3_594m", "pll2_bus", base + 0x100, 3); + hws[IMX6UL_CLK_PLL3_PFD0] = imx_clk_hw_pfd("pll3_pfd0_720m", "pll3_usb_otg", base + 0xf0, 0); + hws[IMX6UL_CLK_PLL3_PFD1] = imx_clk_hw_pfd("pll3_pfd1_540m", "pll3_usb_otg", base + 0xf0, 1); + hws[IMX6UL_CLK_PLL3_PFD2] = imx_clk_hw_pfd("pll3_pfd2_508m", "pll3_usb_otg", base + 0xf0, 2); + hws[IMX6UL_CLK_PLL3_PFD3] = imx_clk_hw_pfd("pll3_pfd3_454m", "pll3_usb_otg", base + 0xf0, 3); + + hws[IMX6UL_CLK_ENET_REF] = clk_hw_register_divider_table(NULL, "enet_ref", "pll6_enet", 0, base + 0xe0, 0, 2, 0, clk_enet_ref_table, &imx_ccm_lock); - clks[IMX6UL_CLK_ENET2_REF] = clk_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0, + hws[IMX6UL_CLK_ENET2_REF] = clk_hw_register_divider_table(NULL, "enet2_ref", "pll6_enet", 0, base + 0xe0, 2, 2, 0, clk_enet_ref_table, &imx_ccm_lock); - clks[IMX6UL_CLK_ENET2_REF_125M] = imx_clk_gate("enet_ref_125m", "enet2_ref", base + 0xe0, 20); - clks[IMX6UL_CLK_ENET_PTP_REF] = imx_clk_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20); - clks[IMX6UL_CLK_ENET_PTP] = imx_clk_gate("enet_ptp", "enet_ptp_ref", base + 0xe0, 21); + hws[IMX6UL_CLK_ENET2_REF_125M] = imx_clk_hw_gate("enet_ref_125m", "enet2_ref", base + 0xe0, 20); + hws[IMX6UL_CLK_ENET_PTP_REF] = imx_clk_hw_fixed_factor("enet_ptp_ref", "pll6_enet", 1, 20); + hws[IMX6UL_CLK_ENET_PTP] = imx_clk_hw_gate("enet_ptp", "enet_ptp_ref", base + 0xe0, 21); - clks[IMX6UL_CLK_PLL4_POST_DIV] = clk_register_divider_table(NULL, "pll4_post_div", "pll4_audio", + hws[IMX6UL_CLK_PLL4_POST_DIV] = clk_hw_register_divider_table(NULL, "pll4_post_div", "pll4_audio", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x70, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6UL_CLK_PLL4_AUDIO_DIV] = clk_register_divider(NULL, "pll4_audio_div", "pll4_post_div", + hws[IMX6UL_CLK_PLL4_AUDIO_DIV] = clk_hw_register_divider(NULL, "pll4_audio_div", "pll4_post_div", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 15, 1, 0, &imx_ccm_lock); - clks[IMX6UL_CLK_PLL5_POST_DIV] = clk_register_divider_table(NULL, "pll5_post_div", "pll5_video", + hws[IMX6UL_CLK_PLL5_POST_DIV] = clk_hw_register_divider_table(NULL, "pll5_post_div", "pll5_video", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xa0, 19, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX6UL_CLK_PLL5_VIDEO_DIV] = clk_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", + hws[IMX6UL_CLK_PLL5_VIDEO_DIV] = clk_hw_register_divider_table(NULL, "pll5_video_div", "pll5_post_div", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x170, 30, 2, 0, video_div_table, &imx_ccm_lock); /* name parent_name mult div */ - clks[IMX6UL_CLK_PLL2_198M] = imx_clk_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); - clks[IMX6UL_CLK_PLL3_80M] = imx_clk_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); - clks[IMX6UL_CLK_PLL3_60M] = imx_clk_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); - clks[IMX6UL_CLK_GPT_3M] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8); + hws[IMX6UL_CLK_PLL2_198M] = imx_clk_hw_fixed_factor("pll2_198m", "pll2_pfd2_396m", 1, 2); + hws[IMX6UL_CLK_PLL3_80M] = imx_clk_hw_fixed_factor("pll3_80m", "pll3_usb_otg", 1, 6); + hws[IMX6UL_CLK_PLL3_60M] = imx_clk_hw_fixed_factor("pll3_60m", "pll3_usb_otg", 1, 8); + hws[IMX6UL_CLK_GPT_3M] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8); np = ccm_node; base = of_iomap(np, 0); WARN_ON(!base); - clks[IMX6UL_CA7_SECONDARY_SEL] = imx_clk_mux("ca7_secondary_sel", base + 0xc, 3, 1, ca7_secondary_sels, ARRAY_SIZE(ca7_secondary_sels)); - clks[IMX6UL_CLK_STEP] = imx_clk_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels)); - clks[IMX6UL_CLK_PLL1_SW] = imx_clk_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0); - clks[IMX6UL_CLK_AXI_ALT_SEL] = imx_clk_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels)); - clks[IMX6UL_CLK_AXI_SEL] = imx_clk_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0); - clks[IMX6UL_CLK_PERIPH_PRE] = imx_clk_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); - clks[IMX6UL_CLK_PERIPH2_PRE] = imx_clk_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels)); - clks[IMX6UL_CLK_PERIPH_CLK2_SEL] = imx_clk_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); - clks[IMX6UL_CLK_PERIPH2_CLK2_SEL] = imx_clk_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); - clks[IMX6UL_CLK_EIM_SLOW_SEL] = imx_clk_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels)); - clks[IMX6UL_CLK_GPMI_SEL] = imx_clk_mux("gpmi_sel", base + 0x1c, 19, 1, gpmi_sels, ARRAY_SIZE(gpmi_sels)); - clks[IMX6UL_CLK_BCH_SEL] = imx_clk_mux("bch_sel", base + 0x1c, 18, 1, bch_sels, ARRAY_SIZE(bch_sels)); - clks[IMX6UL_CLK_USDHC2_SEL] = imx_clk_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6UL_CLK_USDHC1_SEL] = imx_clk_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); - clks[IMX6UL_CLK_SAI3_SEL] = imx_clk_mux("sai3_sel", base + 0x1c, 14, 2, sai_sels, ARRAY_SIZE(sai_sels)); - clks[IMX6UL_CLK_SAI2_SEL] = imx_clk_mux("sai2_sel", base + 0x1c, 12, 2, sai_sels, ARRAY_SIZE(sai_sels)); - clks[IMX6UL_CLK_SAI1_SEL] = imx_clk_mux("sai1_sel", base + 0x1c, 10, 2, sai_sels, ARRAY_SIZE(sai_sels)); - clks[IMX6UL_CLK_QSPI1_SEL] = imx_clk_mux("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels)); - clks[IMX6UL_CLK_PERCLK_SEL] = imx_clk_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels)); - clks[IMX6UL_CLK_CAN_SEL] = imx_clk_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels)); + hws[IMX6UL_CA7_SECONDARY_SEL] = imx_clk_hw_mux("ca7_secondary_sel", base + 0xc, 3, 1, ca7_secondary_sels, ARRAY_SIZE(ca7_secondary_sels)); + hws[IMX6UL_CLK_STEP] = imx_clk_hw_mux("step", base + 0x0c, 8, 1, step_sels, ARRAY_SIZE(step_sels)); + hws[IMX6UL_CLK_PLL1_SW] = imx_clk_hw_mux_flags("pll1_sw", base + 0x0c, 2, 1, pll1_sw_sels, ARRAY_SIZE(pll1_sw_sels), 0); + hws[IMX6UL_CLK_AXI_ALT_SEL] = imx_clk_hw_mux("axi_alt_sel", base + 0x14, 7, 1, axi_alt_sels, ARRAY_SIZE(axi_alt_sels)); + hws[IMX6UL_CLK_AXI_SEL] = imx_clk_hw_mux_flags("axi_sel", base + 0x14, 6, 1, axi_sels, ARRAY_SIZE(axi_sels), 0); + hws[IMX6UL_CLK_PERIPH_PRE] = imx_clk_hw_mux("periph_pre", base + 0x18, 18, 2, periph_pre_sels, ARRAY_SIZE(periph_pre_sels)); + hws[IMX6UL_CLK_PERIPH2_PRE] = imx_clk_hw_mux("periph2_pre", base + 0x18, 21, 2, periph2_pre_sels, ARRAY_SIZE(periph2_pre_sels)); + hws[IMX6UL_CLK_PERIPH_CLK2_SEL] = imx_clk_hw_mux("periph_clk2_sel", base + 0x18, 12, 2, periph_clk2_sels, ARRAY_SIZE(periph_clk2_sels)); + hws[IMX6UL_CLK_PERIPH2_CLK2_SEL] = imx_clk_hw_mux("periph2_clk2_sel", base + 0x18, 20, 1, periph2_clk2_sels, ARRAY_SIZE(periph2_clk2_sels)); + hws[IMX6UL_CLK_EIM_SLOW_SEL] = imx_clk_hw_mux("eim_slow_sel", base + 0x1c, 29, 2, eim_slow_sels, ARRAY_SIZE(eim_slow_sels)); + hws[IMX6UL_CLK_GPMI_SEL] = imx_clk_hw_mux("gpmi_sel", base + 0x1c, 19, 1, gpmi_sels, ARRAY_SIZE(gpmi_sels)); + hws[IMX6UL_CLK_BCH_SEL] = imx_clk_hw_mux("bch_sel", base + 0x1c, 18, 1, bch_sels, ARRAY_SIZE(bch_sels)); + hws[IMX6UL_CLK_USDHC2_SEL] = imx_clk_hw_mux("usdhc2_sel", base + 0x1c, 17, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6UL_CLK_USDHC1_SEL] = imx_clk_hw_mux("usdhc1_sel", base + 0x1c, 16, 1, usdhc_sels, ARRAY_SIZE(usdhc_sels)); + hws[IMX6UL_CLK_SAI3_SEL] = imx_clk_hw_mux("sai3_sel", base + 0x1c, 14, 2, sai_sels, ARRAY_SIZE(sai_sels)); + hws[IMX6UL_CLK_SAI2_SEL] = imx_clk_hw_mux("sai2_sel", base + 0x1c, 12, 2, sai_sels, ARRAY_SIZE(sai_sels)); + hws[IMX6UL_CLK_SAI1_SEL] = imx_clk_hw_mux("sai1_sel", base + 0x1c, 10, 2, sai_sels, ARRAY_SIZE(sai_sels)); + hws[IMX6UL_CLK_QSPI1_SEL] = imx_clk_hw_mux("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels)); + hws[IMX6UL_CLK_PERCLK_SEL] = imx_clk_hw_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels)); + hws[IMX6UL_CLK_CAN_SEL] = imx_clk_hw_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels)); if (clk_on_imx6ull()) - clks[IMX6ULL_CLK_ESAI_SEL] = imx_clk_mux("esai_sel", base + 0x20, 19, 2, esai_sels, ARRAY_SIZE(esai_sels)); - clks[IMX6UL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); - clks[IMX6UL_CLK_ENFC_SEL] = imx_clk_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels, ARRAY_SIZE(enfc_sels)); - clks[IMX6UL_CLK_LDB_DI0_SEL] = imx_clk_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels)); - clks[IMX6UL_CLK_SPDIF_SEL] = imx_clk_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels)); + hws[IMX6ULL_CLK_ESAI_SEL] = imx_clk_hw_mux("esai_sel", base + 0x20, 19, 2, esai_sels, ARRAY_SIZE(esai_sels)); + hws[IMX6UL_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); + hws[IMX6UL_CLK_ENFC_SEL] = imx_clk_hw_mux("enfc_sel", base + 0x2c, 15, 3, enfc_sels, ARRAY_SIZE(enfc_sels)); + hws[IMX6UL_CLK_LDB_DI0_SEL] = imx_clk_hw_mux("ldb_di0_sel", base + 0x2c, 9, 3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels)); + hws[IMX6UL_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels)); if (clk_on_imx6ul()) { - clks[IMX6UL_CLK_SIM_PRE_SEL] = imx_clk_mux("sim_pre_sel", base + 0x34, 15, 3, sim_pre_sels, ARRAY_SIZE(sim_pre_sels)); - clks[IMX6UL_CLK_SIM_SEL] = imx_clk_mux("sim_sel", base + 0x34, 9, 3, sim_sels, ARRAY_SIZE(sim_sels)); + hws[IMX6UL_CLK_SIM_PRE_SEL] = imx_clk_hw_mux("sim_pre_sel", base + 0x34, 15, 3, sim_pre_sels, ARRAY_SIZE(sim_pre_sels)); + hws[IMX6UL_CLK_SIM_SEL] = imx_clk_hw_mux("sim_sel", base + 0x34, 9, 3, sim_sels, ARRAY_SIZE(sim_sels)); } else if (clk_on_imx6ull()) { - clks[IMX6ULL_CLK_EPDC_PRE_SEL] = imx_clk_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels)); - clks[IMX6ULL_CLK_EPDC_SEL] = imx_clk_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels)); + hws[IMX6ULL_CLK_EPDC_PRE_SEL] = imx_clk_hw_mux("epdc_pre_sel", base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels)); + hws[IMX6ULL_CLK_EPDC_SEL] = imx_clk_hw_mux("epdc_sel", base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels)); } - clks[IMX6UL_CLK_ECSPI_SEL] = imx_clk_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); - clks[IMX6UL_CLK_LCDIF_PRE_SEL] = imx_clk_mux_flags("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels), CLK_SET_RATE_PARENT); - clks[IMX6UL_CLK_LCDIF_SEL] = imx_clk_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels)); - - clks[IMX6UL_CLK_LDB_DI0_DIV_SEL] = imx_clk_mux("ldb_di0", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels)); - clks[IMX6UL_CLK_LDB_DI1_DIV_SEL] = imx_clk_mux("ldb_di1", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels)); - - clks[IMX6UL_CLK_CKO1_SEL] = imx_clk_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels)); - clks[IMX6UL_CLK_CKO2_SEL] = imx_clk_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels)); - clks[IMX6UL_CLK_CKO] = imx_clk_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels)); - - clks[IMX6UL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); - clks[IMX6UL_CLK_LDB_DI0_DIV_7] = imx_clk_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7); - clks[IMX6UL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "qspi1_sel", 2, 7); - clks[IMX6UL_CLK_LDB_DI1_DIV_7] = imx_clk_fixed_factor("ldb_di1_div_7", "qspi1_sel", 1, 7); - - clks[IMX6UL_CLK_PERIPH] = imx_clk_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); - clks[IMX6UL_CLK_PERIPH2] = imx_clk_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); - - clks[IMX6UL_CLK_PERIPH_CLK2] = imx_clk_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); - clks[IMX6UL_CLK_PERIPH2_CLK2] = imx_clk_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); - clks[IMX6UL_CLK_IPG] = imx_clk_divider("ipg", "ahb", base + 0x14, 8, 2); - clks[IMX6UL_CLK_LCDIF_PODF] = imx_clk_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3); - clks[IMX6UL_CLK_QSPI1_PDOF] = imx_clk_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3); - clks[IMX6UL_CLK_EIM_SLOW_PODF] = imx_clk_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3); - clks[IMX6UL_CLK_PERCLK] = imx_clk_divider("perclk", "perclk_sel", base + 0x1c, 0, 6); - clks[IMX6UL_CLK_CAN_PODF] = imx_clk_divider("can_podf", "can_sel", base + 0x20, 2, 6); - clks[IMX6UL_CLK_GPMI_PODF] = imx_clk_divider("gpmi_podf", "gpmi_sel", base + 0x24, 22, 3); - clks[IMX6UL_CLK_BCH_PODF] = imx_clk_divider("bch_podf", "bch_sel", base + 0x24, 19, 3); - clks[IMX6UL_CLK_USDHC2_PODF] = imx_clk_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); - clks[IMX6UL_CLK_USDHC1_PODF] = imx_clk_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); - clks[IMX6UL_CLK_UART_PODF] = imx_clk_divider("uart_podf", "uart_sel", base + 0x24, 0, 6); - clks[IMX6UL_CLK_SAI3_PRED] = imx_clk_divider("sai3_pred", "sai3_sel", base + 0x28, 22, 3); - clks[IMX6UL_CLK_SAI3_PODF] = imx_clk_divider("sai3_podf", "sai3_pred", base + 0x28, 16, 6); - clks[IMX6UL_CLK_SAI1_PRED] = imx_clk_divider("sai1_pred", "sai1_sel", base + 0x28, 6, 3); - clks[IMX6UL_CLK_SAI1_PODF] = imx_clk_divider("sai1_podf", "sai1_pred", base + 0x28, 0, 6); + hws[IMX6UL_CLK_ECSPI_SEL] = imx_clk_hw_mux("ecspi_sel", base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels)); + hws[IMX6UL_CLK_LCDIF_PRE_SEL] = imx_clk_hw_mux_flags("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels), CLK_SET_RATE_PARENT); + hws[IMX6UL_CLK_LCDIF_SEL] = imx_clk_hw_mux("lcdif_sel", base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels)); + + hws[IMX6UL_CLK_LDB_DI0_DIV_SEL] = imx_clk_hw_mux("ldb_di0", base + 0x20, 10, 1, ldb_di0_div_sels, ARRAY_SIZE(ldb_di0_div_sels)); + hws[IMX6UL_CLK_LDB_DI1_DIV_SEL] = imx_clk_hw_mux("ldb_di1", base + 0x20, 11, 1, ldb_di1_div_sels, ARRAY_SIZE(ldb_di1_div_sels)); + + hws[IMX6UL_CLK_CKO1_SEL] = imx_clk_hw_mux("cko1_sel", base + 0x60, 0, 4, cko1_sels, ARRAY_SIZE(cko1_sels)); + hws[IMX6UL_CLK_CKO2_SEL] = imx_clk_hw_mux("cko2_sel", base + 0x60, 16, 5, cko2_sels, ARRAY_SIZE(cko2_sels)); + hws[IMX6UL_CLK_CKO] = imx_clk_hw_mux("cko", base + 0x60, 8, 1, cko_sels, ARRAY_SIZE(cko_sels)); + + hws[IMX6UL_CLK_LDB_DI0_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7); + hws[IMX6UL_CLK_LDB_DI0_DIV_7] = imx_clk_hw_fixed_factor("ldb_di0_div_7", "ldb_di0_sel", 1, 7); + hws[IMX6UL_CLK_LDB_DI1_DIV_3_5] = imx_clk_hw_fixed_factor("ldb_di1_div_3_5", "qspi1_sel", 2, 7); + hws[IMX6UL_CLK_LDB_DI1_DIV_7] = imx_clk_hw_fixed_factor("ldb_di1_div_7", "qspi1_sel", 1, 7); + + hws[IMX6UL_CLK_PERIPH] = imx_clk_hw_busy_mux("periph", base + 0x14, 25, 1, base + 0x48, 5, periph_sels, ARRAY_SIZE(periph_sels)); + hws[IMX6UL_CLK_PERIPH2] = imx_clk_hw_busy_mux("periph2", base + 0x14, 26, 1, base + 0x48, 3, periph2_sels, ARRAY_SIZE(periph2_sels)); + + hws[IMX6UL_CLK_PERIPH_CLK2] = imx_clk_hw_divider("periph_clk2", "periph_clk2_sel", base + 0x14, 27, 3); + hws[IMX6UL_CLK_PERIPH2_CLK2] = imx_clk_hw_divider("periph2_clk2", "periph2_clk2_sel", base + 0x14, 0, 3); + hws[IMX6UL_CLK_IPG] = imx_clk_hw_divider("ipg", "ahb", base + 0x14, 8, 2); + hws[IMX6UL_CLK_LCDIF_PODF] = imx_clk_hw_divider("lcdif_podf", "lcdif_pred", base + 0x18, 23, 3); + hws[IMX6UL_CLK_QSPI1_PDOF] = imx_clk_hw_divider("qspi1_podf", "qspi1_sel", base + 0x1c, 26, 3); + hws[IMX6UL_CLK_EIM_SLOW_PODF] = imx_clk_hw_divider("eim_slow_podf", "eim_slow_sel", base + 0x1c, 23, 3); + hws[IMX6UL_CLK_PERCLK] = imx_clk_hw_divider("perclk", "perclk_sel", base + 0x1c, 0, 6); + hws[IMX6UL_CLK_CAN_PODF] = imx_clk_hw_divider("can_podf", "can_sel", base + 0x20, 2, 6); + hws[IMX6UL_CLK_GPMI_PODF] = imx_clk_hw_divider("gpmi_podf", "gpmi_sel", base + 0x24, 22, 3); + hws[IMX6UL_CLK_BCH_PODF] = imx_clk_hw_divider("bch_podf", "bch_sel", base + 0x24, 19, 3); + hws[IMX6UL_CLK_USDHC2_PODF] = imx_clk_hw_divider("usdhc2_podf", "usdhc2_sel", base + 0x24, 16, 3); + hws[IMX6UL_CLK_USDHC1_PODF] = imx_clk_hw_divider("usdhc1_podf", "usdhc1_sel", base + 0x24, 11, 3); + hws[IMX6UL_CLK_UART_PODF] = imx_clk_hw_divider("uart_podf", "uart_sel", base + 0x24, 0, 6); + hws[IMX6UL_CLK_SAI3_PRED] = imx_clk_hw_divider("sai3_pred", "sai3_sel", base + 0x28, 22, 3); + hws[IMX6UL_CLK_SAI3_PODF] = imx_clk_hw_divider("sai3_podf", "sai3_pred", base + 0x28, 16, 6); + hws[IMX6UL_CLK_SAI1_PRED] = imx_clk_hw_divider("sai1_pred", "sai1_sel", base + 0x28, 6, 3); + hws[IMX6UL_CLK_SAI1_PODF] = imx_clk_hw_divider("sai1_podf", "sai1_pred", base + 0x28, 0, 6); if (clk_on_imx6ull()) { - clks[IMX6ULL_CLK_ESAI_PRED] = imx_clk_divider("esai_pred", "esai_sel", base + 0x28, 9, 3); - clks[IMX6ULL_CLK_ESAI_PODF] = imx_clk_divider("esai_podf", "esai_pred", base + 0x28, 25, 3); + hws[IMX6ULL_CLK_ESAI_PRED] = imx_clk_hw_divider("esai_pred", "esai_sel", base + 0x28, 9, 3); + hws[IMX6ULL_CLK_ESAI_PODF] = imx_clk_hw_divider("esai_podf", "esai_pred", base + 0x28, 25, 3); } - clks[IMX6UL_CLK_ENFC_PRED] = imx_clk_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3); - clks[IMX6UL_CLK_ENFC_PODF] = imx_clk_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6); - clks[IMX6UL_CLK_SAI2_PRED] = imx_clk_divider("sai2_pred", "sai2_sel", base + 0x2c, 6, 3); - clks[IMX6UL_CLK_SAI2_PODF] = imx_clk_divider("sai2_podf", "sai2_pred", base + 0x2c, 0, 6); - clks[IMX6UL_CLK_SPDIF_PRED] = imx_clk_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); - clks[IMX6UL_CLK_SPDIF_PODF] = imx_clk_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); + hws[IMX6UL_CLK_ENFC_PRED] = imx_clk_hw_divider("enfc_pred", "enfc_sel", base + 0x2c, 18, 3); + hws[IMX6UL_CLK_ENFC_PODF] = imx_clk_hw_divider("enfc_podf", "enfc_pred", base + 0x2c, 21, 6); + hws[IMX6UL_CLK_SAI2_PRED] = imx_clk_hw_divider("sai2_pred", "sai2_sel", base + 0x2c, 6, 3); + hws[IMX6UL_CLK_SAI2_PODF] = imx_clk_hw_divider("sai2_podf", "sai2_pred", base + 0x2c, 0, 6); + hws[IMX6UL_CLK_SPDIF_PRED] = imx_clk_hw_divider("spdif_pred", "spdif_sel", base + 0x30, 25, 3); + hws[IMX6UL_CLK_SPDIF_PODF] = imx_clk_hw_divider("spdif_podf", "spdif_pred", base + 0x30, 22, 3); if (clk_on_imx6ul()) - clks[IMX6UL_CLK_SIM_PODF] = imx_clk_divider("sim_podf", "sim_pre_sel", base + 0x34, 12, 3); + hws[IMX6UL_CLK_SIM_PODF] = imx_clk_hw_divider("sim_podf", "sim_pre_sel", base + 0x34, 12, 3); else if (clk_on_imx6ull()) - clks[IMX6ULL_CLK_EPDC_PODF] = imx_clk_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3); - clks[IMX6UL_CLK_ECSPI_PODF] = imx_clk_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6); - clks[IMX6UL_CLK_LCDIF_PRED] = imx_clk_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3); - clks[IMX6UL_CLK_CSI_PODF] = imx_clk_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3); + hws[IMX6ULL_CLK_EPDC_PODF] = imx_clk_hw_divider("epdc_podf", "epdc_pre_sel", base + 0x34, 12, 3); + hws[IMX6UL_CLK_ECSPI_PODF] = imx_clk_hw_divider("ecspi_podf", "ecspi_sel", base + 0x38, 19, 6); + hws[IMX6UL_CLK_LCDIF_PRED] = imx_clk_hw_divider("lcdif_pred", "lcdif_pre_sel", base + 0x38, 12, 3); + hws[IMX6UL_CLK_CSI_PODF] = imx_clk_hw_divider("csi_podf", "csi_sel", base + 0x3c, 11, 3); - clks[IMX6UL_CLK_CKO1_PODF] = imx_clk_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3); - clks[IMX6UL_CLK_CKO2_PODF] = imx_clk_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3); + hws[IMX6UL_CLK_CKO1_PODF] = imx_clk_hw_divider("cko1_podf", "cko1_sel", base + 0x60, 4, 3); + hws[IMX6UL_CLK_CKO2_PODF] = imx_clk_hw_divider("cko2_podf", "cko2_sel", base + 0x60, 21, 3); - clks[IMX6UL_CLK_ARM] = imx_clk_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); - clks[IMX6UL_CLK_MMDC_PODF] = imx_clk_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); - clks[IMX6UL_CLK_AXI_PODF] = imx_clk_busy_divider("axi_podf", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0); - clks[IMX6UL_CLK_AHB] = imx_clk_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); + hws[IMX6UL_CLK_ARM] = imx_clk_hw_busy_divider("arm", "pll1_sw", base + 0x10, 0, 3, base + 0x48, 16); + hws[IMX6UL_CLK_MMDC_PODF] = imx_clk_hw_busy_divider("mmdc_podf", "periph2", base + 0x14, 3, 3, base + 0x48, 2); + hws[IMX6UL_CLK_AXI_PODF] = imx_clk_hw_busy_divider("axi_podf", "axi_sel", base + 0x14, 16, 3, base + 0x48, 0); + hws[IMX6UL_CLK_AHB] = imx_clk_hw_busy_divider("ahb", "periph", base + 0x14, 10, 3, base + 0x48, 1); /* CCGR0 */ - clks[IMX6UL_CLK_AIPSTZ1] = imx_clk_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL); - clks[IMX6UL_CLK_AIPSTZ2] = imx_clk_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL); - clks[IMX6UL_CLK_APBHDMA] = imx_clk_gate2("apbh_dma", "bch_podf", base + 0x68, 4); - clks[IMX6UL_CLK_ASRC_IPG] = imx_clk_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc); - clks[IMX6UL_CLK_ASRC_MEM] = imx_clk_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc); + hws[IMX6UL_CLK_AIPSTZ1] = imx_clk_hw_gate2_flags("aips_tz1", "ahb", base + 0x68, 0, CLK_IS_CRITICAL); + hws[IMX6UL_CLK_AIPSTZ2] = imx_clk_hw_gate2_flags("aips_tz2", "ahb", base + 0x68, 2, CLK_IS_CRITICAL); + hws[IMX6UL_CLK_APBHDMA] = imx_clk_hw_gate2("apbh_dma", "bch_podf", base + 0x68, 4); + hws[IMX6UL_CLK_ASRC_IPG] = imx_clk_hw_gate2_shared("asrc_ipg", "ahb", base + 0x68, 6, &share_count_asrc); + hws[IMX6UL_CLK_ASRC_MEM] = imx_clk_hw_gate2_shared("asrc_mem", "ahb", base + 0x68, 6, &share_count_asrc); if (clk_on_imx6ul()) { - clks[IMX6UL_CLK_CAAM_MEM] = imx_clk_gate2("caam_mem", "ahb", base + 0x68, 8); - clks[IMX6UL_CLK_CAAM_ACLK] = imx_clk_gate2("caam_aclk", "ahb", base + 0x68, 10); - clks[IMX6UL_CLK_CAAM_IPG] = imx_clk_gate2("caam_ipg", "ipg", base + 0x68, 12); + hws[IMX6UL_CLK_CAAM_MEM] = imx_clk_hw_gate2("caam_mem", "ahb", base + 0x68, 8); + hws[IMX6UL_CLK_CAAM_ACLK] = imx_clk_hw_gate2("caam_aclk", "ahb", base + 0x68, 10); + hws[IMX6UL_CLK_CAAM_IPG] = imx_clk_hw_gate2("caam_ipg", "ipg", base + 0x68, 12); } else if (clk_on_imx6ull()) { - clks[IMX6ULL_CLK_DCP_CLK] = imx_clk_gate2("dcp", "ahb", base + 0x68, 10); - clks[IMX6UL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x68, 12); - clks[IMX6UL_CLK_ENET_AHB] = imx_clk_gate2("enet_ahb", "ahb", base + 0x68, 12); + hws[IMX6ULL_CLK_DCP_CLK] = imx_clk_hw_gate2("dcp", "ahb", base + 0x68, 10); + hws[IMX6UL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x68, 12); + hws[IMX6UL_CLK_ENET_AHB] = imx_clk_hw_gate2("enet_ahb", "ahb", base + 0x68, 12); } - clks[IMX6UL_CLK_CAN1_IPG] = imx_clk_gate2("can1_ipg", "ipg", base + 0x68, 14); - clks[IMX6UL_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_podf", base + 0x68, 16); - clks[IMX6UL_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18); - clks[IMX6UL_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_podf", base + 0x68, 20); - clks[IMX6UL_CLK_GPT2_BUS] = imx_clk_gate2("gpt2_bus", "perclk", base + 0x68, 24); - clks[IMX6UL_CLK_GPT2_SERIAL] = imx_clk_gate2("gpt2_serial", "perclk", base + 0x68, 26); - clks[IMX6UL_CLK_UART2_IPG] = imx_clk_gate2("uart2_ipg", "ipg", base + 0x68, 28); - clks[IMX6UL_CLK_UART2_SERIAL] = imx_clk_gate2("uart2_serial", "uart_podf", base + 0x68, 28); + hws[IMX6UL_CLK_CAN1_IPG] = imx_clk_hw_gate2("can1_ipg", "ipg", base + 0x68, 14); + hws[IMX6UL_CLK_CAN1_SERIAL] = imx_clk_hw_gate2("can1_serial", "can_podf", base + 0x68, 16); + hws[IMX6UL_CLK_CAN2_IPG] = imx_clk_hw_gate2("can2_ipg", "ipg", base + 0x68, 18); + hws[IMX6UL_CLK_CAN2_SERIAL] = imx_clk_hw_gate2("can2_serial", "can_podf", base + 0x68, 20); + hws[IMX6UL_CLK_GPT2_BUS] = imx_clk_hw_gate2("gpt2_bus", "perclk", base + 0x68, 24); + hws[IMX6UL_CLK_GPT2_SERIAL] = imx_clk_hw_gate2("gpt2_serial", "perclk", base + 0x68, 26); + hws[IMX6UL_CLK_UART2_IPG] = imx_clk_hw_gate2("uart2_ipg", "ipg", base + 0x68, 28); + hws[IMX6UL_CLK_UART2_SERIAL] = imx_clk_hw_gate2("uart2_serial", "uart_podf", base + 0x68, 28); if (clk_on_imx6ull()) - clks[IMX6UL_CLK_AIPSTZ3] = imx_clk_gate2("aips_tz3", "ahb", base + 0x80, 18); - clks[IMX6UL_CLK_GPIO2] = imx_clk_gate2("gpio2", "ipg", base + 0x68, 30); + hws[IMX6UL_CLK_AIPSTZ3] = imx_clk_hw_gate2("aips_tz3", "ahb", base + 0x80, 18); + hws[IMX6UL_CLK_GPIO2] = imx_clk_hw_gate2("gpio2", "ipg", base + 0x68, 30); /* CCGR1 */ - clks[IMX6UL_CLK_ECSPI1] = imx_clk_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0); - clks[IMX6UL_CLK_ECSPI2] = imx_clk_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2); - clks[IMX6UL_CLK_ECSPI3] = imx_clk_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4); - clks[IMX6UL_CLK_ECSPI4] = imx_clk_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6); - clks[IMX6UL_CLK_ADC2] = imx_clk_gate2("adc2", "ipg", base + 0x6c, 8); - clks[IMX6UL_CLK_UART3_IPG] = imx_clk_gate2("uart3_ipg", "ipg", base + 0x6c, 10); - clks[IMX6UL_CLK_UART3_SERIAL] = imx_clk_gate2("uart3_serial", "uart_podf", base + 0x6c, 10); - clks[IMX6UL_CLK_EPIT1] = imx_clk_gate2("epit1", "perclk", base + 0x6c, 12); - clks[IMX6UL_CLK_EPIT2] = imx_clk_gate2("epit2", "perclk", base + 0x6c, 14); - clks[IMX6UL_CLK_ADC1] = imx_clk_gate2("adc1", "ipg", base + 0x6c, 16); - clks[IMX6UL_CLK_GPT1_BUS] = imx_clk_gate2("gpt1_bus", "perclk", base + 0x6c, 20); - clks[IMX6UL_CLK_GPT1_SERIAL] = imx_clk_gate2("gpt1_serial", "perclk", base + 0x6c, 22); - clks[IMX6UL_CLK_UART4_IPG] = imx_clk_gate2("uart4_ipg", "ipg", base + 0x6c, 24); - clks[IMX6UL_CLK_UART4_SERIAL] = imx_clk_gate2("uart4_serial", "uart_podf", base + 0x6c, 24); - clks[IMX6UL_CLK_GPIO1] = imx_clk_gate2("gpio1", "ipg", base + 0x6c, 26); - clks[IMX6UL_CLK_GPIO5] = imx_clk_gate2("gpio5", "ipg", base + 0x6c, 30); + hws[IMX6UL_CLK_ECSPI1] = imx_clk_hw_gate2("ecspi1", "ecspi_podf", base + 0x6c, 0); + hws[IMX6UL_CLK_ECSPI2] = imx_clk_hw_gate2("ecspi2", "ecspi_podf", base + 0x6c, 2); + hws[IMX6UL_CLK_ECSPI3] = imx_clk_hw_gate2("ecspi3", "ecspi_podf", base + 0x6c, 4); + hws[IMX6UL_CLK_ECSPI4] = imx_clk_hw_gate2("ecspi4", "ecspi_podf", base + 0x6c, 6); + hws[IMX6UL_CLK_ADC2] = imx_clk_hw_gate2("adc2", "ipg", base + 0x6c, 8); + hws[IMX6UL_CLK_UART3_IPG] = imx_clk_hw_gate2("uart3_ipg", "ipg", base + 0x6c, 10); + hws[IMX6UL_CLK_UART3_SERIAL] = imx_clk_hw_gate2("uart3_serial", "uart_podf", base + 0x6c, 10); + hws[IMX6UL_CLK_EPIT1] = imx_clk_hw_gate2("epit1", "perclk", base + 0x6c, 12); + hws[IMX6UL_CLK_EPIT2] = imx_clk_hw_gate2("epit2", "perclk", base + 0x6c, 14); + hws[IMX6UL_CLK_ADC1] = imx_clk_hw_gate2("adc1", "ipg", base + 0x6c, 16); + hws[IMX6UL_CLK_GPT1_BUS] = imx_clk_hw_gate2("gpt1_bus", "perclk", base + 0x6c, 20); + hws[IMX6UL_CLK_GPT1_SERIAL] = imx_clk_hw_gate2("gpt1_serial", "perclk", base + 0x6c, 22); + hws[IMX6UL_CLK_UART4_IPG] = imx_clk_hw_gate2("uart4_ipg", "ipg", base + 0x6c, 24); + hws[IMX6UL_CLK_UART4_SERIAL] = imx_clk_hw_gate2("uart4_serial", "uart_podf", base + 0x6c, 24); + hws[IMX6UL_CLK_GPIO1] = imx_clk_hw_gate2("gpio1", "ipg", base + 0x6c, 26); + hws[IMX6UL_CLK_GPIO5] = imx_clk_hw_gate2("gpio5", "ipg", base + 0x6c, 30); /* CCGR2 */ if (clk_on_imx6ull()) { - clks[IMX6ULL_CLK_ESAI_EXTAL] = imx_clk_gate2_shared("esai_extal", "esai_podf", base + 0x70, 0, &share_count_esai); - clks[IMX6ULL_CLK_ESAI_IPG] = imx_clk_gate2_shared("esai_ipg", "ahb", base + 0x70, 0, &share_count_esai); - clks[IMX6ULL_CLK_ESAI_MEM] = imx_clk_gate2_shared("esai_mem", "ahb", base + 0x70, 0, &share_count_esai); + hws[IMX6ULL_CLK_ESAI_EXTAL] = imx_clk_hw_gate2_shared("esai_extal", "esai_podf", base + 0x70, 0, &share_count_esai); + hws[IMX6ULL_CLK_ESAI_IPG] = imx_clk_hw_gate2_shared("esai_ipg", "ahb", base + 0x70, 0, &share_count_esai); + hws[IMX6ULL_CLK_ESAI_MEM] = imx_clk_hw_gate2_shared("esai_mem", "ahb", base + 0x70, 0, &share_count_esai); } - clks[IMX6UL_CLK_CSI] = imx_clk_gate2("csi", "csi_podf", base + 0x70, 2); - clks[IMX6UL_CLK_I2C1] = imx_clk_gate2("i2c1", "perclk", base + 0x70, 6); - clks[IMX6UL_CLK_I2C2] = imx_clk_gate2("i2c2", "perclk", base + 0x70, 8); - clks[IMX6UL_CLK_I2C3] = imx_clk_gate2("i2c3", "perclk", base + 0x70, 10); - clks[IMX6UL_CLK_OCOTP] = imx_clk_gate2("ocotp", "ipg", base + 0x70, 12); - clks[IMX6UL_CLK_IOMUXC] = imx_clk_gate2("iomuxc", "lcdif_podf", base + 0x70, 14); - clks[IMX6UL_CLK_GPIO3] = imx_clk_gate2("gpio3", "ipg", base + 0x70, 26); - clks[IMX6UL_CLK_LCDIF_APB] = imx_clk_gate2("lcdif_apb", "axi", base + 0x70, 28); - clks[IMX6UL_CLK_PXP] = imx_clk_gate2("pxp", "axi", base + 0x70, 30); + hws[IMX6UL_CLK_CSI] = imx_clk_hw_gate2("csi", "csi_podf", base + 0x70, 2); + hws[IMX6UL_CLK_I2C1] = imx_clk_hw_gate2("i2c1", "perclk", base + 0x70, 6); + hws[IMX6UL_CLK_I2C2] = imx_clk_hw_gate2("i2c2", "perclk", base + 0x70, 8); + hws[IMX6UL_CLK_I2C3] = imx_clk_hw_gate2("i2c3", "perclk", base + 0x70, 10); + hws[IMX6UL_CLK_OCOTP] = imx_clk_hw_gate2("ocotp", "ipg", base + 0x70, 12); + hws[IMX6UL_CLK_IOMUXC] = imx_clk_hw_gate2("iomuxc", "lcdif_podf", base + 0x70, 14); + hws[IMX6UL_CLK_GPIO3] = imx_clk_hw_gate2("gpio3", "ipg", base + 0x70, 26); + hws[IMX6UL_CLK_LCDIF_APB] = imx_clk_hw_gate2("lcdif_apb", "axi", base + 0x70, 28); + hws[IMX6UL_CLK_PXP] = imx_clk_hw_gate2("pxp", "axi", base + 0x70, 30); /* CCGR3 */ - clks[IMX6UL_CLK_UART5_IPG] = imx_clk_gate2("uart5_ipg", "ipg", base + 0x74, 2); - clks[IMX6UL_CLK_UART5_SERIAL] = imx_clk_gate2("uart5_serial", "uart_podf", base + 0x74, 2); + hws[IMX6UL_CLK_UART5_IPG] = imx_clk_hw_gate2("uart5_ipg", "ipg", base + 0x74, 2); + hws[IMX6UL_CLK_UART5_SERIAL] = imx_clk_hw_gate2("uart5_serial", "uart_podf", base + 0x74, 2); if (clk_on_imx6ul()) { - clks[IMX6UL_CLK_ENET] = imx_clk_gate2("enet", "ipg", base + 0x74, 4); - clks[IMX6UL_CLK_ENET_AHB] = imx_clk_gate2("enet_ahb", "ahb", base + 0x74, 4); + hws[IMX6UL_CLK_ENET] = imx_clk_hw_gate2("enet", "ipg", base + 0x74, 4); + hws[IMX6UL_CLK_ENET_AHB] = imx_clk_hw_gate2("enet_ahb", "ahb", base + 0x74, 4); } else if (clk_on_imx6ull()) { - clks[IMX6ULL_CLK_EPDC_ACLK] = imx_clk_gate2("epdc_aclk", "axi", base + 0x74, 4); - clks[IMX6ULL_CLK_EPDC_PIX] = imx_clk_gate2("epdc_pix", "epdc_podf", base + 0x74, 4); + hws[IMX6ULL_CLK_EPDC_ACLK] = imx_clk_hw_gate2("epdc_aclk", "axi", base + 0x74, 4); + hws[IMX6ULL_CLK_EPDC_PIX] = imx_clk_hw_gate2("epdc_pix", "epdc_podf", base + 0x74, 4); } - clks[IMX6UL_CLK_UART6_IPG] = imx_clk_gate2("uart6_ipg", "ipg", base + 0x74, 6); - clks[IMX6UL_CLK_UART6_SERIAL] = imx_clk_gate2("uart6_serial", "uart_podf", base + 0x74, 6); - clks[IMX6UL_CLK_LCDIF_PIX] = imx_clk_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10); - clks[IMX6UL_CLK_GPIO4] = imx_clk_gate2("gpio4", "ipg", base + 0x74, 12); - clks[IMX6UL_CLK_QSPI] = imx_clk_gate2("qspi1", "qspi1_podf", base + 0x74, 14); - clks[IMX6UL_CLK_WDOG1] = imx_clk_gate2("wdog1", "ipg", base + 0x74, 16); - clks[IMX6UL_CLK_MMDC_P0_FAST] = imx_clk_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL); - clks[IMX6UL_CLK_MMDC_P0_IPG] = imx_clk_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); - clks[IMX6UL_CLK_MMDC_P1_IPG] = imx_clk_gate2("mmdc_p1_ipg", "ipg", base + 0x74, 26); - clks[IMX6UL_CLK_AXI] = imx_clk_gate_flags("axi", "axi_podf", base + 0x74, 28, CLK_IS_CRITICAL); + hws[IMX6UL_CLK_UART6_IPG] = imx_clk_hw_gate2("uart6_ipg", "ipg", base + 0x74, 6); + hws[IMX6UL_CLK_UART6_SERIAL] = imx_clk_hw_gate2("uart6_serial", "uart_podf", base + 0x74, 6); + hws[IMX6UL_CLK_LCDIF_PIX] = imx_clk_hw_gate2("lcdif_pix", "lcdif_podf", base + 0x74, 10); + hws[IMX6UL_CLK_GPIO4] = imx_clk_hw_gate2("gpio4", "ipg", base + 0x74, 12); + hws[IMX6UL_CLK_QSPI] = imx_clk_hw_gate2("qspi1", "qspi1_podf", base + 0x74, 14); + hws[IMX6UL_CLK_WDOG1] = imx_clk_hw_gate2("wdog1", "ipg", base + 0x74, 16); + hws[IMX6UL_CLK_MMDC_P0_FAST] = imx_clk_hw_gate_flags("mmdc_p0_fast", "mmdc_podf", base + 0x74, 20, CLK_IS_CRITICAL); + hws[IMX6UL_CLK_MMDC_P0_IPG] = imx_clk_hw_gate2_flags("mmdc_p0_ipg", "ipg", base + 0x74, 24, CLK_IS_CRITICAL); + hws[IMX6UL_CLK_MMDC_P1_IPG] = imx_clk_hw_gate2_flags("mmdc_p1_ipg", "ipg", base + 0x74, 26, CLK_IS_CRITICAL); + hws[IMX6UL_CLK_AXI] = imx_clk_hw_gate_flags("axi", "axi_podf", base + 0x74, 28, CLK_IS_CRITICAL); /* CCGR4 */ - clks[IMX6UL_CLK_PER_BCH] = imx_clk_gate2("per_bch", "bch_podf", base + 0x78, 12); - clks[IMX6UL_CLK_PWM1] = imx_clk_gate2("pwm1", "perclk", base + 0x78, 16); - clks[IMX6UL_CLK_PWM2] = imx_clk_gate2("pwm2", "perclk", base + 0x78, 18); - clks[IMX6UL_CLK_PWM3] = imx_clk_gate2("pwm3", "perclk", base + 0x78, 20); - clks[IMX6UL_CLK_PWM4] = imx_clk_gate2("pwm4", "perclk", base + 0x78, 22); - clks[IMX6UL_CLK_GPMI_BCH_APB] = imx_clk_gate2("gpmi_bch_apb", "bch_podf", base + 0x78, 24); - clks[IMX6UL_CLK_GPMI_BCH] = imx_clk_gate2("gpmi_bch", "gpmi_podf", base + 0x78, 26); - clks[IMX6UL_CLK_GPMI_IO] = imx_clk_gate2("gpmi_io", "enfc_podf", base + 0x78, 28); - clks[IMX6UL_CLK_GPMI_APB] = imx_clk_gate2("gpmi_apb", "bch_podf", base + 0x78, 30); + hws[IMX6UL_CLK_PER_BCH] = imx_clk_hw_gate2("per_bch", "bch_podf", base + 0x78, 12); + hws[IMX6UL_CLK_PWM1] = imx_clk_hw_gate2("pwm1", "perclk", base + 0x78, 16); + hws[IMX6UL_CLK_PWM2] = imx_clk_hw_gate2("pwm2", "perclk", base + 0x78, 18); + hws[IMX6UL_CLK_PWM3] = imx_clk_hw_gate2("pwm3", "perclk", base + 0x78, 20); + hws[IMX6UL_CLK_PWM4] = imx_clk_hw_gate2("pwm4", "perclk", base + 0x78, 22); + hws[IMX6UL_CLK_GPMI_BCH_APB] = imx_clk_hw_gate2("gpmi_bch_apb", "bch_podf", base + 0x78, 24); + hws[IMX6UL_CLK_GPMI_BCH] = imx_clk_hw_gate2("gpmi_bch", "gpmi_podf", base + 0x78, 26); + hws[IMX6UL_CLK_GPMI_IO] = imx_clk_hw_gate2("gpmi_io", "enfc_podf", base + 0x78, 28); + hws[IMX6UL_CLK_GPMI_APB] = imx_clk_hw_gate2("gpmi_apb", "bch_podf", base + 0x78, 30); /* CCGR5 */ - clks[IMX6UL_CLK_ROM] = imx_clk_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); - clks[IMX6UL_CLK_SDMA] = imx_clk_gate2("sdma", "ahb", base + 0x7c, 6); - clks[IMX6UL_CLK_KPP] = imx_clk_gate2("kpp", "ipg", base + 0x7c, 8); - clks[IMX6UL_CLK_WDOG2] = imx_clk_gate2("wdog2", "ipg", base + 0x7c, 10); - clks[IMX6UL_CLK_SPBA] = imx_clk_gate2("spba", "ipg", base + 0x7c, 12); - clks[IMX6UL_CLK_SPDIF] = imx_clk_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio); - clks[IMX6UL_CLK_SPDIF_GCLK] = imx_clk_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio); - clks[IMX6UL_CLK_SAI3] = imx_clk_gate2_shared("sai3", "sai3_podf", base + 0x7c, 22, &share_count_sai3); - clks[IMX6UL_CLK_SAI3_IPG] = imx_clk_gate2_shared("sai3_ipg", "ipg", base + 0x7c, 22, &share_count_sai3); - clks[IMX6UL_CLK_UART1_IPG] = imx_clk_gate2("uart1_ipg", "ipg", base + 0x7c, 24); - clks[IMX6UL_CLK_UART1_SERIAL] = imx_clk_gate2("uart1_serial", "uart_podf", base + 0x7c, 24); - clks[IMX6UL_CLK_UART7_IPG] = imx_clk_gate2("uart7_ipg", "ipg", base + 0x7c, 26); - clks[IMX6UL_CLK_UART7_SERIAL] = imx_clk_gate2("uart7_serial", "uart_podf", base + 0x7c, 26); - clks[IMX6UL_CLK_SAI1] = imx_clk_gate2_shared("sai1", "sai1_podf", base + 0x7c, 28, &share_count_sai1); - clks[IMX6UL_CLK_SAI1_IPG] = imx_clk_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1); - clks[IMX6UL_CLK_SAI2] = imx_clk_gate2_shared("sai2", "sai2_podf", base + 0x7c, 30, &share_count_sai2); - clks[IMX6UL_CLK_SAI2_IPG] = imx_clk_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2); + hws[IMX6UL_CLK_ROM] = imx_clk_hw_gate2_flags("rom", "ahb", base + 0x7c, 0, CLK_IS_CRITICAL); + hws[IMX6UL_CLK_SDMA] = imx_clk_hw_gate2("sdma", "ahb", base + 0x7c, 6); + hws[IMX6UL_CLK_KPP] = imx_clk_hw_gate2("kpp", "ipg", base + 0x7c, 8); + hws[IMX6UL_CLK_WDOG2] = imx_clk_hw_gate2("wdog2", "ipg", base + 0x7c, 10); + hws[IMX6UL_CLK_SPBA] = imx_clk_hw_gate2("spba", "ipg", base + 0x7c, 12); + hws[IMX6UL_CLK_SPDIF] = imx_clk_hw_gate2_shared("spdif", "spdif_podf", base + 0x7c, 14, &share_count_audio); + hws[IMX6UL_CLK_SPDIF_GCLK] = imx_clk_hw_gate2_shared("spdif_gclk", "ipg", base + 0x7c, 14, &share_count_audio); + hws[IMX6UL_CLK_SAI3] = imx_clk_hw_gate2_shared("sai3", "sai3_podf", base + 0x7c, 22, &share_count_sai3); + hws[IMX6UL_CLK_SAI3_IPG] = imx_clk_hw_gate2_shared("sai3_ipg", "ipg", base + 0x7c, 22, &share_count_sai3); + hws[IMX6UL_CLK_UART1_IPG] = imx_clk_hw_gate2("uart1_ipg", "ipg", base + 0x7c, 24); + hws[IMX6UL_CLK_UART1_SERIAL] = imx_clk_hw_gate2("uart1_serial", "uart_podf", base + 0x7c, 24); + hws[IMX6UL_CLK_UART7_IPG] = imx_clk_hw_gate2("uart7_ipg", "ipg", base + 0x7c, 26); + hws[IMX6UL_CLK_UART7_SERIAL] = imx_clk_hw_gate2("uart7_serial", "uart_podf", base + 0x7c, 26); + hws[IMX6UL_CLK_SAI1] = imx_clk_hw_gate2_shared("sai1", "sai1_podf", base + 0x7c, 28, &share_count_sai1); + hws[IMX6UL_CLK_SAI1_IPG] = imx_clk_hw_gate2_shared("sai1_ipg", "ipg", base + 0x7c, 28, &share_count_sai1); + hws[IMX6UL_CLK_SAI2] = imx_clk_hw_gate2_shared("sai2", "sai2_podf", base + 0x7c, 30, &share_count_sai2); + hws[IMX6UL_CLK_SAI2_IPG] = imx_clk_hw_gate2_shared("sai2_ipg", "ipg", base + 0x7c, 30, &share_count_sai2); /* CCGR6 */ - clks[IMX6UL_CLK_USBOH3] = imx_clk_gate2("usboh3", "ipg", base + 0x80, 0); - clks[IMX6UL_CLK_USDHC1] = imx_clk_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); - clks[IMX6UL_CLK_USDHC2] = imx_clk_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); + hws[IMX6UL_CLK_USBOH3] = imx_clk_hw_gate2("usboh3", "ipg", base + 0x80, 0); + hws[IMX6UL_CLK_USDHC1] = imx_clk_hw_gate2("usdhc1", "usdhc1_podf", base + 0x80, 2); + hws[IMX6UL_CLK_USDHC2] = imx_clk_hw_gate2("usdhc2", "usdhc2_podf", base + 0x80, 4); if (clk_on_imx6ul()) { - clks[IMX6UL_CLK_SIM1] = imx_clk_gate2("sim1", "sim_sel", base + 0x80, 6); - clks[IMX6UL_CLK_SIM2] = imx_clk_gate2("sim2", "sim_sel", base + 0x80, 8); + hws[IMX6UL_CLK_SIM1] = imx_clk_hw_gate2("sim1", "sim_sel", base + 0x80, 6); + hws[IMX6UL_CLK_SIM2] = imx_clk_hw_gate2("sim2", "sim_sel", base + 0x80, 8); } - clks[IMX6UL_CLK_EIM] = imx_clk_gate2("eim", "eim_slow_podf", base + 0x80, 10); - clks[IMX6UL_CLK_PWM8] = imx_clk_gate2("pwm8", "perclk", base + 0x80, 16); - clks[IMX6UL_CLK_UART8_IPG] = imx_clk_gate2("uart8_ipg", "ipg", base + 0x80, 14); - clks[IMX6UL_CLK_UART8_SERIAL] = imx_clk_gate2("uart8_serial", "uart_podf", base + 0x80, 14); - clks[IMX6UL_CLK_WDOG3] = imx_clk_gate2("wdog3", "ipg", base + 0x80, 20); - clks[IMX6UL_CLK_I2C4] = imx_clk_gate2("i2c4", "perclk", base + 0x80, 24); - clks[IMX6UL_CLK_PWM5] = imx_clk_gate2("pwm5", "perclk", base + 0x80, 26); - clks[IMX6UL_CLK_PWM6] = imx_clk_gate2("pwm6", "perclk", base + 0x80, 28); - clks[IMX6UL_CLK_PWM7] = imx_clk_gate2("pwm7", "perclk", base + 0x80, 30); + hws[IMX6UL_CLK_EIM] = imx_clk_hw_gate2("eim", "eim_slow_podf", base + 0x80, 10); + hws[IMX6UL_CLK_PWM8] = imx_clk_hw_gate2("pwm8", "perclk", base + 0x80, 16); + hws[IMX6UL_CLK_UART8_IPG] = imx_clk_hw_gate2("uart8_ipg", "ipg", base + 0x80, 14); + hws[IMX6UL_CLK_UART8_SERIAL] = imx_clk_hw_gate2("uart8_serial", "uart_podf", base + 0x80, 14); + hws[IMX6UL_CLK_WDOG3] = imx_clk_hw_gate2("wdog3", "ipg", base + 0x80, 20); + hws[IMX6UL_CLK_I2C4] = imx_clk_hw_gate2("i2c4", "perclk", base + 0x80, 24); + hws[IMX6UL_CLK_PWM5] = imx_clk_hw_gate2("pwm5", "perclk", base + 0x80, 26); + hws[IMX6UL_CLK_PWM6] = imx_clk_hw_gate2("pwm6", "perclk", base + 0x80, 28); + hws[IMX6UL_CLK_PWM7] = imx_clk_hw_gate2("pwm7", "perclk", base + 0x80, 30); /* CCOSR */ - clks[IMX6UL_CLK_CKO1] = imx_clk_gate("cko1", "cko1_podf", base + 0x60, 7); - clks[IMX6UL_CLK_CKO2] = imx_clk_gate("cko2", "cko2_podf", base + 0x60, 24); + hws[IMX6UL_CLK_CKO1] = imx_clk_hw_gate("cko1", "cko1_podf", base + 0x60, 7); + hws[IMX6UL_CLK_CKO2] = imx_clk_hw_gate("cko2", "cko2_podf", base + 0x60, 24); /* mask handshake of mmdc */ - writel_relaxed(BM_CCM_CCDR_MMDC_CH0_MASK, base + CCDR); + imx_mmdc_mask_handshake(base, 0); - imx_check_clocks(clks, ARRAY_SIZE(clks)); + imx_check_clk_hws(hws, IMX6UL_CLK_END); - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); + of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); /* * Lower the AHB clock rate before changing the parent clock source, @@ -473,39 +477,39 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node) * AXI clock rate, so we need to lower AHB rate first to make sure at * any time, AHB rate is <= 133MHz. */ - clk_set_rate(clks[IMX6UL_CLK_AHB], 99000000); + clk_set_rate(hws[IMX6UL_CLK_AHB]->clk, 99000000); /* Change periph_pre clock to pll2_bus to adjust AXI rate to 264MHz */ - clk_set_parent(clks[IMX6UL_CLK_PERIPH_CLK2_SEL], clks[IMX6UL_CLK_OSC]); - clk_set_parent(clks[IMX6UL_CLK_PERIPH], clks[IMX6UL_CLK_PERIPH_CLK2]); - clk_set_parent(clks[IMX6UL_CLK_PERIPH_PRE], clks[IMX6UL_CLK_PLL2_BUS]); - clk_set_parent(clks[IMX6UL_CLK_PERIPH], clks[IMX6UL_CLK_PERIPH_PRE]); + clk_set_parent(hws[IMX6UL_CLK_PERIPH_CLK2_SEL]->clk, hws[IMX6UL_CLK_OSC]->clk); + clk_set_parent(hws[IMX6UL_CLK_PERIPH]->clk, hws[IMX6UL_CLK_PERIPH_CLK2]->clk); + clk_set_parent(hws[IMX6UL_CLK_PERIPH_PRE]->clk, hws[IMX6UL_CLK_PLL2_BUS]->clk); + clk_set_parent(hws[IMX6UL_CLK_PERIPH]->clk, hws[IMX6UL_CLK_PERIPH_PRE]->clk); /* Make sure AHB rate is 132MHz */ - clk_set_rate(clks[IMX6UL_CLK_AHB], 132000000); + clk_set_rate(hws[IMX6UL_CLK_AHB]->clk, 132000000); /* set perclk to from OSC */ - clk_set_parent(clks[IMX6UL_CLK_PERCLK_SEL], clks[IMX6UL_CLK_OSC]); + clk_set_parent(hws[IMX6UL_CLK_PERCLK_SEL]->clk, hws[IMX6UL_CLK_OSC]->clk); - clk_set_rate(clks[IMX6UL_CLK_ENET_REF], 50000000); - clk_set_rate(clks[IMX6UL_CLK_ENET2_REF], 50000000); - clk_set_rate(clks[IMX6UL_CLK_CSI], 24000000); + clk_set_rate(hws[IMX6UL_CLK_ENET_REF]->clk, 50000000); + clk_set_rate(hws[IMX6UL_CLK_ENET2_REF]->clk, 50000000); + clk_set_rate(hws[IMX6UL_CLK_CSI]->clk, 24000000); if (clk_on_imx6ull()) - clk_prepare_enable(clks[IMX6UL_CLK_AIPSTZ3]); + clk_prepare_enable(hws[IMX6UL_CLK_AIPSTZ3]->clk); if (IS_ENABLED(CONFIG_USB_MXS_PHY)) { - clk_prepare_enable(clks[IMX6UL_CLK_USBPHY1_GATE]); - clk_prepare_enable(clks[IMX6UL_CLK_USBPHY2_GATE]); + clk_prepare_enable(hws[IMX6UL_CLK_USBPHY1_GATE]->clk); + clk_prepare_enable(hws[IMX6UL_CLK_USBPHY2_GATE]->clk); } - clk_set_parent(clks[IMX6UL_CLK_CAN_SEL], clks[IMX6UL_CLK_PLL3_60M]); + clk_set_parent(hws[IMX6UL_CLK_CAN_SEL]->clk, hws[IMX6UL_CLK_PLL3_60M]->clk); if (clk_on_imx6ul()) - clk_set_parent(clks[IMX6UL_CLK_SIM_PRE_SEL], clks[IMX6UL_CLK_PLL3_USB_OTG]); + clk_set_parent(hws[IMX6UL_CLK_SIM_PRE_SEL]->clk, hws[IMX6UL_CLK_PLL3_USB_OTG]->clk); else if (clk_on_imx6ull()) - clk_set_parent(clks[IMX6ULL_CLK_EPDC_PRE_SEL], clks[IMX6UL_CLK_PLL3_PFD2]); + clk_set_parent(hws[IMX6ULL_CLK_EPDC_PRE_SEL]->clk, hws[IMX6UL_CLK_PLL3_PFD2]->clk); - clk_set_parent(clks[IMX6UL_CLK_ENFC_SEL], clks[IMX6UL_CLK_PLL2_PFD2]); + clk_set_parent(hws[IMX6UL_CLK_ENFC_SEL]->clk, hws[IMX6UL_CLK_PLL2_PFD2]->clk); } CLK_OF_DECLARE(imx6ul, "fsl,imx6ul-ccm", imx6ul_clocks_init); diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c index 8f3aa994c8f7..fbea774ef687 100644 --- a/drivers/clk/imx/clk-imx7d.c +++ b/drivers/clk/imx/clk-imx7d.c @@ -6,6 +6,7 @@ #include <dt-bindings/clock/imx7d-clock.h> #include <linux/clk.h> #include <linux/clkdev.h> +#include <linux/clk-provider.h> #include <linux/err.h> #include <linux/init.h> #include <linux/io.h> @@ -39,7 +40,6 @@ static const struct clk_div_table post_div_table[] = { { } }; -static struct clk *clks[IMX7D_CLK_END]; static const char *arm_a7_sel[] = { "osc", "pll_arm_main_clk", "pll_enet_500m_clk", "pll_dram_main_clk", "pll_sys_main_clk", "pll_sys_pfd0_392m_clk", "pll_audio_post_div", @@ -373,517 +373,533 @@ static const char *pll_enet_bypass_sel[] = { "pll_enet_main", "pll_enet_main_src static const char *pll_audio_bypass_sel[] = { "pll_audio_main", "pll_audio_main_src", }; static const char *pll_video_bypass_sel[] = { "pll_video_main", "pll_video_main_src", }; -static struct clk_onecell_data clk_data; - -static struct clk ** const uart_clks[] __initconst = { - &clks[IMX7D_UART1_ROOT_CLK], - &clks[IMX7D_UART2_ROOT_CLK], - &clks[IMX7D_UART3_ROOT_CLK], - &clks[IMX7D_UART4_ROOT_CLK], - &clks[IMX7D_UART5_ROOT_CLK], - &clks[IMX7D_UART6_ROOT_CLK], - &clks[IMX7D_UART7_ROOT_CLK], - NULL +static struct clk_hw **hws; +static struct clk_hw_onecell_data *clk_hw_data; + +static const int uart_clk_ids[] __initconst = { + IMX7D_UART1_ROOT_CLK, + IMX7D_UART2_ROOT_CLK, + IMX7D_UART3_ROOT_CLK, + IMX7D_UART4_ROOT_CLK, + IMX7D_UART5_ROOT_CLK, + IMX7D_UART6_ROOT_CLK, + IMX7D_UART7_ROOT_CLK, }; +static struct clk **uart_clks[ARRAY_SIZE(uart_clk_ids) + 1] __initdata; + static void __init imx7d_clocks_init(struct device_node *ccm_node) { struct device_node *np; void __iomem *base; + int i; + + clk_hw_data = kzalloc(struct_size(clk_hw_data, hws, + IMX7D_CLK_END), GFP_KERNEL); + if (WARN_ON(!clk_hw_data)) + return; + + clk_hw_data->num = IMX7D_CLK_END; + hws = clk_hw_data->hws; - clks[IMX7D_CLK_DUMMY] = imx_clk_fixed("dummy", 0); - clks[IMX7D_OSC_24M_CLK] = of_clk_get_by_name(ccm_node, "osc"); - clks[IMX7D_CKIL] = of_clk_get_by_name(ccm_node, "ckil"); + hws[IMX7D_CLK_DUMMY] = imx_clk_hw_fixed("dummy", 0); + hws[IMX7D_OSC_24M_CLK] = __clk_get_hw(of_clk_get_by_name(ccm_node, "osc")); + hws[IMX7D_CKIL] = __clk_get_hw(of_clk_get_by_name(ccm_node, "ckil")); np = of_find_compatible_node(NULL, NULL, "fsl,imx7d-anatop"); base = of_iomap(np, 0); WARN_ON(!base); of_node_put(np); - clks[IMX7D_PLL_ARM_MAIN_SRC] = imx_clk_mux("pll_arm_main_src", base + 0x60, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); - clks[IMX7D_PLL_DRAM_MAIN_SRC] = imx_clk_mux("pll_dram_main_src", base + 0x70, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); - clks[IMX7D_PLL_SYS_MAIN_SRC] = imx_clk_mux("pll_sys_main_src", base + 0xb0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); - clks[IMX7D_PLL_ENET_MAIN_SRC] = imx_clk_mux("pll_enet_main_src", base + 0xe0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); - clks[IMX7D_PLL_AUDIO_MAIN_SRC] = imx_clk_mux("pll_audio_main_src", base + 0xf0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); - clks[IMX7D_PLL_VIDEO_MAIN_SRC] = imx_clk_mux("pll_video_main_src", base + 0x130, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); - - clks[IMX7D_PLL_ARM_MAIN] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll_arm_main", "osc", base + 0x60, 0x7f); - clks[IMX7D_PLL_DRAM_MAIN] = imx_clk_pllv3(IMX_PLLV3_DDR_IMX7, "pll_dram_main", "osc", base + 0x70, 0x7f); - clks[IMX7D_PLL_SYS_MAIN] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll_sys_main", "osc", base + 0xb0, 0x1); - clks[IMX7D_PLL_ENET_MAIN] = imx_clk_pllv3(IMX_PLLV3_ENET_IMX7, "pll_enet_main", "osc", base + 0xe0, 0x0); - clks[IMX7D_PLL_AUDIO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV_IMX7, "pll_audio_main", "osc", base + 0xf0, 0x7f); - clks[IMX7D_PLL_VIDEO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV_IMX7, "pll_video_main", "osc", base + 0x130, 0x7f); - - clks[IMX7D_PLL_ARM_MAIN_BYPASS] = imx_clk_mux_flags("pll_arm_main_bypass", base + 0x60, 16, 1, pll_arm_bypass_sel, ARRAY_SIZE(pll_arm_bypass_sel), CLK_SET_RATE_PARENT); - clks[IMX7D_PLL_DRAM_MAIN_BYPASS] = imx_clk_mux_flags("pll_dram_main_bypass", base + 0x70, 16, 1, pll_dram_bypass_sel, ARRAY_SIZE(pll_dram_bypass_sel), CLK_SET_RATE_PARENT); - clks[IMX7D_PLL_SYS_MAIN_BYPASS] = imx_clk_mux_flags("pll_sys_main_bypass", base + 0xb0, 16, 1, pll_sys_bypass_sel, ARRAY_SIZE(pll_sys_bypass_sel), CLK_SET_RATE_PARENT); - clks[IMX7D_PLL_ENET_MAIN_BYPASS] = imx_clk_mux_flags("pll_enet_main_bypass", base + 0xe0, 16, 1, pll_enet_bypass_sel, ARRAY_SIZE(pll_enet_bypass_sel), CLK_SET_RATE_PARENT); - clks[IMX7D_PLL_AUDIO_MAIN_BYPASS] = imx_clk_mux_flags("pll_audio_main_bypass", base + 0xf0, 16, 1, pll_audio_bypass_sel, ARRAY_SIZE(pll_audio_bypass_sel), CLK_SET_RATE_PARENT); - clks[IMX7D_PLL_VIDEO_MAIN_BYPASS] = imx_clk_mux_flags("pll_video_main_bypass", base + 0x130, 16, 1, pll_video_bypass_sel, ARRAY_SIZE(pll_video_bypass_sel), CLK_SET_RATE_PARENT); - - clks[IMX7D_PLL_ARM_MAIN_CLK] = imx_clk_gate("pll_arm_main_clk", "pll_arm_main_bypass", base + 0x60, 13); - clks[IMX7D_PLL_DRAM_MAIN_CLK] = imx_clk_gate("pll_dram_main_clk", "pll_dram_test_div", base + 0x70, 13); - clks[IMX7D_PLL_SYS_MAIN_CLK] = imx_clk_gate("pll_sys_main_clk", "pll_sys_main_bypass", base + 0xb0, 13); - clks[IMX7D_PLL_AUDIO_MAIN_CLK] = imx_clk_gate("pll_audio_main_clk", "pll_audio_main_bypass", base + 0xf0, 13); - clks[IMX7D_PLL_VIDEO_MAIN_CLK] = imx_clk_gate("pll_video_main_clk", "pll_video_main_bypass", base + 0x130, 13); - - clks[IMX7D_PLL_DRAM_TEST_DIV] = clk_register_divider_table(NULL, "pll_dram_test_div", "pll_dram_main_bypass", + hws[IMX7D_PLL_ARM_MAIN_SRC] = imx_clk_hw_mux("pll_arm_main_src", base + 0x60, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); + hws[IMX7D_PLL_DRAM_MAIN_SRC] = imx_clk_hw_mux("pll_dram_main_src", base + 0x70, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); + hws[IMX7D_PLL_SYS_MAIN_SRC] = imx_clk_hw_mux("pll_sys_main_src", base + 0xb0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); + hws[IMX7D_PLL_ENET_MAIN_SRC] = imx_clk_hw_mux("pll_enet_main_src", base + 0xe0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); + hws[IMX7D_PLL_AUDIO_MAIN_SRC] = imx_clk_hw_mux("pll_audio_main_src", base + 0xf0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); + hws[IMX7D_PLL_VIDEO_MAIN_SRC] = imx_clk_hw_mux("pll_video_main_src", base + 0x130, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); + + hws[IMX7D_PLL_ARM_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_SYS, "pll_arm_main", "osc", base + 0x60, 0x7f); + hws[IMX7D_PLL_DRAM_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_DDR_IMX7, "pll_dram_main", "osc", base + 0x70, 0x7f); + hws[IMX7D_PLL_SYS_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_GENERIC, "pll_sys_main", "osc", base + 0xb0, 0x1); + hws[IMX7D_PLL_ENET_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_ENET_IMX7, "pll_enet_main", "osc", base + 0xe0, 0x0); + hws[IMX7D_PLL_AUDIO_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_AV_IMX7, "pll_audio_main", "osc", base + 0xf0, 0x7f); + hws[IMX7D_PLL_VIDEO_MAIN] = imx_clk_hw_pllv3(IMX_PLLV3_AV_IMX7, "pll_video_main", "osc", base + 0x130, 0x7f); + + hws[IMX7D_PLL_ARM_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_arm_main_bypass", base + 0x60, 16, 1, pll_arm_bypass_sel, ARRAY_SIZE(pll_arm_bypass_sel), CLK_SET_RATE_PARENT); + hws[IMX7D_PLL_DRAM_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_dram_main_bypass", base + 0x70, 16, 1, pll_dram_bypass_sel, ARRAY_SIZE(pll_dram_bypass_sel), CLK_SET_RATE_PARENT); + hws[IMX7D_PLL_SYS_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_sys_main_bypass", base + 0xb0, 16, 1, pll_sys_bypass_sel, ARRAY_SIZE(pll_sys_bypass_sel), CLK_SET_RATE_PARENT); + hws[IMX7D_PLL_ENET_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_enet_main_bypass", base + 0xe0, 16, 1, pll_enet_bypass_sel, ARRAY_SIZE(pll_enet_bypass_sel), CLK_SET_RATE_PARENT); + hws[IMX7D_PLL_AUDIO_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_audio_main_bypass", base + 0xf0, 16, 1, pll_audio_bypass_sel, ARRAY_SIZE(pll_audio_bypass_sel), CLK_SET_RATE_PARENT); + hws[IMX7D_PLL_VIDEO_MAIN_BYPASS] = imx_clk_hw_mux_flags("pll_video_main_bypass", base + 0x130, 16, 1, pll_video_bypass_sel, ARRAY_SIZE(pll_video_bypass_sel), CLK_SET_RATE_PARENT); + + hws[IMX7D_PLL_ARM_MAIN_CLK] = imx_clk_hw_gate("pll_arm_main_clk", "pll_arm_main_bypass", base + 0x60, 13); + hws[IMX7D_PLL_DRAM_MAIN_CLK] = imx_clk_hw_gate("pll_dram_main_clk", "pll_dram_test_div", base + 0x70, 13); + hws[IMX7D_PLL_SYS_MAIN_CLK] = imx_clk_hw_gate("pll_sys_main_clk", "pll_sys_main_bypass", base + 0xb0, 13); + hws[IMX7D_PLL_AUDIO_MAIN_CLK] = imx_clk_hw_gate("pll_audio_main_clk", "pll_audio_main_bypass", base + 0xf0, 13); + hws[IMX7D_PLL_VIDEO_MAIN_CLK] = imx_clk_hw_gate("pll_video_main_clk", "pll_video_main_bypass", base + 0x130, 13); + + hws[IMX7D_PLL_DRAM_TEST_DIV] = clk_hw_register_divider_table(NULL, "pll_dram_test_div", "pll_dram_main_bypass", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x70, 21, 2, 0, test_div_table, &imx_ccm_lock); - clks[IMX7D_PLL_AUDIO_TEST_DIV] = clk_register_divider_table(NULL, "pll_audio_test_div", "pll_audio_main_clk", + hws[IMX7D_PLL_AUDIO_TEST_DIV] = clk_hw_register_divider_table(NULL, "pll_audio_test_div", "pll_audio_main_clk", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xf0, 19, 2, 0, test_div_table, &imx_ccm_lock); - clks[IMX7D_PLL_AUDIO_POST_DIV] = clk_register_divider_table(NULL, "pll_audio_post_div", "pll_audio_test_div", + hws[IMX7D_PLL_AUDIO_POST_DIV] = clk_hw_register_divider_table(NULL, "pll_audio_post_div", "pll_audio_test_div", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xf0, 22, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX7D_PLL_VIDEO_TEST_DIV] = clk_register_divider_table(NULL, "pll_video_test_div", "pll_video_main_clk", + hws[IMX7D_PLL_VIDEO_TEST_DIV] = clk_hw_register_divider_table(NULL, "pll_video_test_div", "pll_video_main_clk", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x130, 19, 2, 0, test_div_table, &imx_ccm_lock); - clks[IMX7D_PLL_VIDEO_POST_DIV] = clk_register_divider_table(NULL, "pll_video_post_div", "pll_video_test_div", + hws[IMX7D_PLL_VIDEO_POST_DIV] = clk_hw_register_divider_table(NULL, "pll_video_post_div", "pll_video_test_div", CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0x130, 22, 2, 0, post_div_table, &imx_ccm_lock); - clks[IMX7D_PLL_SYS_PFD0_392M_CLK] = imx_clk_pfd("pll_sys_pfd0_392m_clk", "pll_sys_main_clk", base + 0xc0, 0); - clks[IMX7D_PLL_SYS_PFD1_332M_CLK] = imx_clk_pfd("pll_sys_pfd1_332m_clk", "pll_sys_main_clk", base + 0xc0, 1); - clks[IMX7D_PLL_SYS_PFD2_270M_CLK] = imx_clk_pfd("pll_sys_pfd2_270m_clk", "pll_sys_main_clk", base + 0xc0, 2); - - clks[IMX7D_PLL_SYS_PFD3_CLK] = imx_clk_pfd("pll_sys_pfd3_clk", "pll_sys_main_clk", base + 0xc0, 3); - clks[IMX7D_PLL_SYS_PFD4_CLK] = imx_clk_pfd("pll_sys_pfd4_clk", "pll_sys_main_clk", base + 0xd0, 0); - clks[IMX7D_PLL_SYS_PFD5_CLK] = imx_clk_pfd("pll_sys_pfd5_clk", "pll_sys_main_clk", base + 0xd0, 1); - clks[IMX7D_PLL_SYS_PFD6_CLK] = imx_clk_pfd("pll_sys_pfd6_clk", "pll_sys_main_clk", base + 0xd0, 2); - clks[IMX7D_PLL_SYS_PFD7_CLK] = imx_clk_pfd("pll_sys_pfd7_clk", "pll_sys_main_clk", base + 0xd0, 3); - - clks[IMX7D_PLL_SYS_MAIN_480M] = imx_clk_fixed_factor("pll_sys_main_480m", "pll_sys_main_clk", 1, 1); - clks[IMX7D_PLL_SYS_MAIN_240M] = imx_clk_fixed_factor("pll_sys_main_240m", "pll_sys_main_clk", 1, 2); - clks[IMX7D_PLL_SYS_MAIN_120M] = imx_clk_fixed_factor("pll_sys_main_120m", "pll_sys_main_clk", 1, 4); - clks[IMX7D_PLL_DRAM_MAIN_533M] = imx_clk_fixed_factor("pll_dram_533m", "pll_dram_main_clk", 1, 2); - - clks[IMX7D_PLL_SYS_MAIN_480M_CLK] = imx_clk_gate_dis_flags("pll_sys_main_480m_clk", "pll_sys_main_480m", base + 0xb0, 4, CLK_IS_CRITICAL); - clks[IMX7D_PLL_SYS_MAIN_240M_CLK] = imx_clk_gate_dis("pll_sys_main_240m_clk", "pll_sys_main_240m", base + 0xb0, 5); - clks[IMX7D_PLL_SYS_MAIN_120M_CLK] = imx_clk_gate_dis("pll_sys_main_120m_clk", "pll_sys_main_120m", base + 0xb0, 6); - clks[IMX7D_PLL_DRAM_MAIN_533M_CLK] = imx_clk_gate("pll_dram_533m_clk", "pll_dram_533m", base + 0x70, 12); - - clks[IMX7D_PLL_SYS_PFD0_196M] = imx_clk_fixed_factor("pll_sys_pfd0_196m", "pll_sys_pfd0_392m_clk", 1, 2); - clks[IMX7D_PLL_SYS_PFD1_166M] = imx_clk_fixed_factor("pll_sys_pfd1_166m", "pll_sys_pfd1_332m_clk", 1, 2); - clks[IMX7D_PLL_SYS_PFD2_135M] = imx_clk_fixed_factor("pll_sys_pfd2_135m", "pll_sys_pfd2_270m_clk", 1, 2); - - clks[IMX7D_PLL_SYS_PFD0_196M_CLK] = imx_clk_gate_dis("pll_sys_pfd0_196m_clk", "pll_sys_pfd0_196m", base + 0xb0, 26); - clks[IMX7D_PLL_SYS_PFD1_166M_CLK] = imx_clk_gate_dis("pll_sys_pfd1_166m_clk", "pll_sys_pfd1_166m", base + 0xb0, 27); - clks[IMX7D_PLL_SYS_PFD2_135M_CLK] = imx_clk_gate_dis("pll_sys_pfd2_135m_clk", "pll_sys_pfd2_135m", base + 0xb0, 28); - - clks[IMX7D_PLL_ENET_MAIN_CLK] = imx_clk_fixed_factor("pll_enet_main_clk", "pll_enet_main_bypass", 1, 1); - clks[IMX7D_PLL_ENET_MAIN_500M] = imx_clk_fixed_factor("pll_enet_500m", "pll_enet_main_clk", 1, 2); - clks[IMX7D_PLL_ENET_MAIN_250M] = imx_clk_fixed_factor("pll_enet_250m", "pll_enet_main_clk", 1, 4); - clks[IMX7D_PLL_ENET_MAIN_125M] = imx_clk_fixed_factor("pll_enet_125m", "pll_enet_main_clk", 1, 8); - clks[IMX7D_PLL_ENET_MAIN_100M] = imx_clk_fixed_factor("pll_enet_100m", "pll_enet_main_clk", 1, 10); - clks[IMX7D_PLL_ENET_MAIN_50M] = imx_clk_fixed_factor("pll_enet_50m", "pll_enet_main_clk", 1, 20); - clks[IMX7D_PLL_ENET_MAIN_40M] = imx_clk_fixed_factor("pll_enet_40m", "pll_enet_main_clk", 1, 25); - clks[IMX7D_PLL_ENET_MAIN_25M] = imx_clk_fixed_factor("pll_enet_25m", "pll_enet_main_clk", 1, 40); - - clks[IMX7D_PLL_ENET_MAIN_500M_CLK] = imx_clk_gate("pll_enet_500m_clk", "pll_enet_500m", base + 0xe0, 12); - clks[IMX7D_PLL_ENET_MAIN_250M_CLK] = imx_clk_gate("pll_enet_250m_clk", "pll_enet_250m", base + 0xe0, 11); - clks[IMX7D_PLL_ENET_MAIN_125M_CLK] = imx_clk_gate("pll_enet_125m_clk", "pll_enet_125m", base + 0xe0, 10); - clks[IMX7D_PLL_ENET_MAIN_100M_CLK] = imx_clk_gate("pll_enet_100m_clk", "pll_enet_100m", base + 0xe0, 9); - clks[IMX7D_PLL_ENET_MAIN_50M_CLK] = imx_clk_gate("pll_enet_50m_clk", "pll_enet_50m", base + 0xe0, 8); - clks[IMX7D_PLL_ENET_MAIN_40M_CLK] = imx_clk_gate("pll_enet_40m_clk", "pll_enet_40m", base + 0xe0, 7); - clks[IMX7D_PLL_ENET_MAIN_25M_CLK] = imx_clk_gate("pll_enet_25m_clk", "pll_enet_25m", base + 0xe0, 6); - - clks[IMX7D_LVDS1_OUT_SEL] = imx_clk_mux("lvds1_sel", base + 0x170, 0, 5, lvds1_sel, ARRAY_SIZE(lvds1_sel)); - clks[IMX7D_LVDS1_OUT_CLK] = imx_clk_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x170, 5, BIT(6)); + hws[IMX7D_PLL_SYS_PFD0_392M_CLK] = imx_clk_hw_pfd("pll_sys_pfd0_392m_clk", "pll_sys_main_clk", base + 0xc0, 0); + hws[IMX7D_PLL_SYS_PFD1_332M_CLK] = imx_clk_hw_pfd("pll_sys_pfd1_332m_clk", "pll_sys_main_clk", base + 0xc0, 1); + hws[IMX7D_PLL_SYS_PFD2_270M_CLK] = imx_clk_hw_pfd("pll_sys_pfd2_270m_clk", "pll_sys_main_clk", base + 0xc0, 2); + + hws[IMX7D_PLL_SYS_PFD3_CLK] = imx_clk_hw_pfd("pll_sys_pfd3_clk", "pll_sys_main_clk", base + 0xc0, 3); + hws[IMX7D_PLL_SYS_PFD4_CLK] = imx_clk_hw_pfd("pll_sys_pfd4_clk", "pll_sys_main_clk", base + 0xd0, 0); + hws[IMX7D_PLL_SYS_PFD5_CLK] = imx_clk_hw_pfd("pll_sys_pfd5_clk", "pll_sys_main_clk", base + 0xd0, 1); + hws[IMX7D_PLL_SYS_PFD6_CLK] = imx_clk_hw_pfd("pll_sys_pfd6_clk", "pll_sys_main_clk", base + 0xd0, 2); + hws[IMX7D_PLL_SYS_PFD7_CLK] = imx_clk_hw_pfd("pll_sys_pfd7_clk", "pll_sys_main_clk", base + 0xd0, 3); + + hws[IMX7D_PLL_SYS_MAIN_480M] = imx_clk_hw_fixed_factor("pll_sys_main_480m", "pll_sys_main_clk", 1, 1); + hws[IMX7D_PLL_SYS_MAIN_240M] = imx_clk_hw_fixed_factor("pll_sys_main_240m", "pll_sys_main_clk", 1, 2); + hws[IMX7D_PLL_SYS_MAIN_120M] = imx_clk_hw_fixed_factor("pll_sys_main_120m", "pll_sys_main_clk", 1, 4); + hws[IMX7D_PLL_DRAM_MAIN_533M] = imx_clk_hw_fixed_factor("pll_dram_533m", "pll_dram_main_clk", 1, 2); + + hws[IMX7D_PLL_SYS_MAIN_480M_CLK] = imx_clk_hw_gate_dis_flags("pll_sys_main_480m_clk", "pll_sys_main_480m", base + 0xb0, 4, CLK_IS_CRITICAL); + hws[IMX7D_PLL_SYS_MAIN_240M_CLK] = imx_clk_hw_gate_dis("pll_sys_main_240m_clk", "pll_sys_main_240m", base + 0xb0, 5); + hws[IMX7D_PLL_SYS_MAIN_120M_CLK] = imx_clk_hw_gate_dis("pll_sys_main_120m_clk", "pll_sys_main_120m", base + 0xb0, 6); + hws[IMX7D_PLL_DRAM_MAIN_533M_CLK] = imx_clk_hw_gate("pll_dram_533m_clk", "pll_dram_533m", base + 0x70, 12); + + hws[IMX7D_PLL_SYS_PFD0_196M] = imx_clk_hw_fixed_factor("pll_sys_pfd0_196m", "pll_sys_pfd0_392m_clk", 1, 2); + hws[IMX7D_PLL_SYS_PFD1_166M] = imx_clk_hw_fixed_factor("pll_sys_pfd1_166m", "pll_sys_pfd1_332m_clk", 1, 2); + hws[IMX7D_PLL_SYS_PFD2_135M] = imx_clk_hw_fixed_factor("pll_sys_pfd2_135m", "pll_sys_pfd2_270m_clk", 1, 2); + + hws[IMX7D_PLL_SYS_PFD0_196M_CLK] = imx_clk_hw_gate_dis("pll_sys_pfd0_196m_clk", "pll_sys_pfd0_196m", base + 0xb0, 26); + hws[IMX7D_PLL_SYS_PFD1_166M_CLK] = imx_clk_hw_gate_dis("pll_sys_pfd1_166m_clk", "pll_sys_pfd1_166m", base + 0xb0, 27); + hws[IMX7D_PLL_SYS_PFD2_135M_CLK] = imx_clk_hw_gate_dis("pll_sys_pfd2_135m_clk", "pll_sys_pfd2_135m", base + 0xb0, 28); + + hws[IMX7D_PLL_ENET_MAIN_CLK] = imx_clk_hw_fixed_factor("pll_enet_main_clk", "pll_enet_main_bypass", 1, 1); + hws[IMX7D_PLL_ENET_MAIN_500M] = imx_clk_hw_fixed_factor("pll_enet_500m", "pll_enet_main_clk", 1, 2); + hws[IMX7D_PLL_ENET_MAIN_250M] = imx_clk_hw_fixed_factor("pll_enet_250m", "pll_enet_main_clk", 1, 4); + hws[IMX7D_PLL_ENET_MAIN_125M] = imx_clk_hw_fixed_factor("pll_enet_125m", "pll_enet_main_clk", 1, 8); + hws[IMX7D_PLL_ENET_MAIN_100M] = imx_clk_hw_fixed_factor("pll_enet_100m", "pll_enet_main_clk", 1, 10); + hws[IMX7D_PLL_ENET_MAIN_50M] = imx_clk_hw_fixed_factor("pll_enet_50m", "pll_enet_main_clk", 1, 20); + hws[IMX7D_PLL_ENET_MAIN_40M] = imx_clk_hw_fixed_factor("pll_enet_40m", "pll_enet_main_clk", 1, 25); + hws[IMX7D_PLL_ENET_MAIN_25M] = imx_clk_hw_fixed_factor("pll_enet_25m", "pll_enet_main_clk", 1, 40); + + hws[IMX7D_PLL_ENET_MAIN_500M_CLK] = imx_clk_hw_gate("pll_enet_500m_clk", "pll_enet_500m", base + 0xe0, 12); + hws[IMX7D_PLL_ENET_MAIN_250M_CLK] = imx_clk_hw_gate("pll_enet_250m_clk", "pll_enet_250m", base + 0xe0, 11); + hws[IMX7D_PLL_ENET_MAIN_125M_CLK] = imx_clk_hw_gate("pll_enet_125m_clk", "pll_enet_125m", base + 0xe0, 10); + hws[IMX7D_PLL_ENET_MAIN_100M_CLK] = imx_clk_hw_gate("pll_enet_100m_clk", "pll_enet_100m", base + 0xe0, 9); + hws[IMX7D_PLL_ENET_MAIN_50M_CLK] = imx_clk_hw_gate("pll_enet_50m_clk", "pll_enet_50m", base + 0xe0, 8); + hws[IMX7D_PLL_ENET_MAIN_40M_CLK] = imx_clk_hw_gate("pll_enet_40m_clk", "pll_enet_40m", base + 0xe0, 7); + hws[IMX7D_PLL_ENET_MAIN_25M_CLK] = imx_clk_hw_gate("pll_enet_25m_clk", "pll_enet_25m", base + 0xe0, 6); + + hws[IMX7D_LVDS1_OUT_SEL] = imx_clk_hw_mux("lvds1_sel", base + 0x170, 0, 5, lvds1_sel, ARRAY_SIZE(lvds1_sel)); + hws[IMX7D_LVDS1_OUT_CLK] = imx_clk_hw_gate_exclusive("lvds1_out", "lvds1_sel", base + 0x170, 5, BIT(6)); np = ccm_node; base = of_iomap(np, 0); WARN_ON(!base); - clks[IMX7D_ARM_A7_ROOT_SRC] = imx_clk_mux2("arm_a7_src", base + 0x8000, 24, 3, arm_a7_sel, ARRAY_SIZE(arm_a7_sel)); - clks[IMX7D_ARM_M4_ROOT_SRC] = imx_clk_mux2("arm_m4_src", base + 0x8080, 24, 3, arm_m4_sel, ARRAY_SIZE(arm_m4_sel)); - clks[IMX7D_MAIN_AXI_ROOT_SRC] = imx_clk_mux2("axi_src", base + 0x8800, 24, 3, axi_sel, ARRAY_SIZE(axi_sel)); - clks[IMX7D_DISP_AXI_ROOT_SRC] = imx_clk_mux2("disp_axi_src", base + 0x8880, 24, 3, disp_axi_sel, ARRAY_SIZE(disp_axi_sel)); - clks[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_mux2("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel)); - clks[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_mux2("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel)); - clks[IMX7D_AHB_CHANNEL_ROOT_SRC] = imx_clk_mux2("ahb_src", base + 0x9000, 24, 3, ahb_channel_sel, ARRAY_SIZE(ahb_channel_sel)); - clks[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_mux2("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel)); - clks[IMX7D_DRAM_ROOT_SRC] = imx_clk_mux2("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel)); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_mux2("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel)); - clks[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_mux2("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel)); - clks[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_mux2("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel)); - clks[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_mux2("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel)); - clks[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_mux2("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel)); - clks[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_mux2("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel)); - clks[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_mux2("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel)); - clks[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_mux2("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel)); - clks[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_mux2("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel)); - clks[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_mux2("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel)); - clks[IMX7D_SAI1_ROOT_SRC] = imx_clk_mux2("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel)); - clks[IMX7D_SAI2_ROOT_SRC] = imx_clk_mux2("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel)); - clks[IMX7D_SAI3_ROOT_SRC] = imx_clk_mux2("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel)); - clks[IMX7D_SPDIF_ROOT_SRC] = imx_clk_mux2("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel)); - clks[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_mux2("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel)); - clks[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_mux2("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel)); - clks[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_mux2("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel)); - clks[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_mux2("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel)); - clks[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_mux2("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel)); - clks[IMX7D_EIM_ROOT_SRC] = imx_clk_mux2("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel)); - clks[IMX7D_NAND_ROOT_SRC] = imx_clk_mux2("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel)); - clks[IMX7D_QSPI_ROOT_SRC] = imx_clk_mux2("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel)); - clks[IMX7D_USDHC1_ROOT_SRC] = imx_clk_mux2("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel)); - clks[IMX7D_USDHC2_ROOT_SRC] = imx_clk_mux2("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel)); - clks[IMX7D_USDHC3_ROOT_SRC] = imx_clk_mux2("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel)); - clks[IMX7D_CAN1_ROOT_SRC] = imx_clk_mux2("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel)); - clks[IMX7D_CAN2_ROOT_SRC] = imx_clk_mux2("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel)); - clks[IMX7D_I2C1_ROOT_SRC] = imx_clk_mux2("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel)); - clks[IMX7D_I2C2_ROOT_SRC] = imx_clk_mux2("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel)); - clks[IMX7D_I2C3_ROOT_SRC] = imx_clk_mux2("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel)); - clks[IMX7D_I2C4_ROOT_SRC] = imx_clk_mux2("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel)); - clks[IMX7D_UART1_ROOT_SRC] = imx_clk_mux2("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel)); - clks[IMX7D_UART2_ROOT_SRC] = imx_clk_mux2("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel)); - clks[IMX7D_UART3_ROOT_SRC] = imx_clk_mux2("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel)); - clks[IMX7D_UART4_ROOT_SRC] = imx_clk_mux2("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel)); - clks[IMX7D_UART5_ROOT_SRC] = imx_clk_mux2("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel)); - clks[IMX7D_UART6_ROOT_SRC] = imx_clk_mux2("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel)); - clks[IMX7D_UART7_ROOT_SRC] = imx_clk_mux2("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel)); - clks[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_mux2("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel)); - clks[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_mux2("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel)); - clks[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_mux2("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel)); - clks[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_mux2("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel)); - clks[IMX7D_PWM1_ROOT_SRC] = imx_clk_mux2("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel)); - clks[IMX7D_PWM2_ROOT_SRC] = imx_clk_mux2("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel)); - clks[IMX7D_PWM3_ROOT_SRC] = imx_clk_mux2("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel)); - clks[IMX7D_PWM4_ROOT_SRC] = imx_clk_mux2("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel)); - clks[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_mux2("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel)); - clks[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_mux2("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel)); - clks[IMX7D_SIM1_ROOT_SRC] = imx_clk_mux2("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel)); - clks[IMX7D_SIM2_ROOT_SRC] = imx_clk_mux2("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel)); - clks[IMX7D_GPT1_ROOT_SRC] = imx_clk_mux2("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel)); - clks[IMX7D_GPT2_ROOT_SRC] = imx_clk_mux2("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel)); - clks[IMX7D_GPT3_ROOT_SRC] = imx_clk_mux2("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel)); - clks[IMX7D_GPT4_ROOT_SRC] = imx_clk_mux2("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel)); - clks[IMX7D_TRACE_ROOT_SRC] = imx_clk_mux2("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel)); - clks[IMX7D_WDOG_ROOT_SRC] = imx_clk_mux2("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel)); - clks[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_mux2("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel)); - clks[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_mux2("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel)); - clks[IMX7D_WRCLK_ROOT_SRC] = imx_clk_mux2("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel)); - clks[IMX7D_CLKO1_ROOT_SRC] = imx_clk_mux2("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel)); - clks[IMX7D_CLKO2_ROOT_SRC] = imx_clk_mux2("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel)); - - clks[IMX7D_ARM_A7_ROOT_CG] = imx_clk_gate3("arm_a7_cg", "arm_a7_src", base + 0x8000, 28); - clks[IMX7D_ARM_M4_ROOT_CG] = imx_clk_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); - clks[IMX7D_MAIN_AXI_ROOT_CG] = imx_clk_gate3("axi_cg", "axi_src", base + 0x8800, 28); - clks[IMX7D_DISP_AXI_ROOT_CG] = imx_clk_gate3("disp_axi_cg", "disp_axi_src", base + 0x8880, 28); - clks[IMX7D_ENET_AXI_ROOT_CG] = imx_clk_gate3("enet_axi_cg", "enet_axi_src", base + 0x8900, 28); - clks[IMX7D_NAND_USDHC_BUS_ROOT_CG] = imx_clk_gate3("nand_usdhc_cg", "nand_usdhc_src", base + 0x8980, 28); - clks[IMX7D_AHB_CHANNEL_ROOT_CG] = imx_clk_gate3("ahb_cg", "ahb_src", base + 0x9000, 28); - clks[IMX7D_DRAM_PHYM_ROOT_CG] = imx_clk_gate3("dram_phym_cg", "dram_phym_src", base + 0x9800, 28); - clks[IMX7D_DRAM_ROOT_CG] = imx_clk_gate3("dram_cg", "dram_src", base + 0x9880, 28); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_CG] = imx_clk_gate3("dram_phym_alt_cg", "dram_phym_alt_src", base + 0xa000, 28); - clks[IMX7D_DRAM_ALT_ROOT_CG] = imx_clk_gate3("dram_alt_cg", "dram_alt_src", base + 0xa080, 28); - clks[IMX7D_USB_HSIC_ROOT_CG] = imx_clk_gate3("usb_hsic_cg", "usb_hsic_src", base + 0xa100, 28); - clks[IMX7D_PCIE_CTRL_ROOT_CG] = imx_clk_gate3("pcie_ctrl_cg", "pcie_ctrl_src", base + 0xa180, 28); - clks[IMX7D_PCIE_PHY_ROOT_CG] = imx_clk_gate3("pcie_phy_cg", "pcie_phy_src", base + 0xa200, 28); - clks[IMX7D_EPDC_PIXEL_ROOT_CG] = imx_clk_gate3("epdc_pixel_cg", "epdc_pixel_src", base + 0xa280, 28); - clks[IMX7D_LCDIF_PIXEL_ROOT_CG] = imx_clk_gate3("lcdif_pixel_cg", "lcdif_pixel_src", base + 0xa300, 28); - clks[IMX7D_MIPI_DSI_ROOT_CG] = imx_clk_gate3("mipi_dsi_cg", "mipi_dsi_src", base + 0xa380, 28); - clks[IMX7D_MIPI_CSI_ROOT_CG] = imx_clk_gate3("mipi_csi_cg", "mipi_csi_src", base + 0xa400, 28); - clks[IMX7D_MIPI_DPHY_ROOT_CG] = imx_clk_gate3("mipi_dphy_cg", "mipi_dphy_src", base + 0xa480, 28); - clks[IMX7D_SAI1_ROOT_CG] = imx_clk_gate3("sai1_cg", "sai1_src", base + 0xa500, 28); - clks[IMX7D_SAI2_ROOT_CG] = imx_clk_gate3("sai2_cg", "sai2_src", base + 0xa580, 28); - clks[IMX7D_SAI3_ROOT_CG] = imx_clk_gate3("sai3_cg", "sai3_src", base + 0xa600, 28); - clks[IMX7D_SPDIF_ROOT_CG] = imx_clk_gate3("spdif_cg", "spdif_src", base + 0xa680, 28); - clks[IMX7D_ENET1_REF_ROOT_CG] = imx_clk_gate3("enet1_ref_cg", "enet1_ref_src", base + 0xa700, 28); - clks[IMX7D_ENET1_TIME_ROOT_CG] = imx_clk_gate3("enet1_time_cg", "enet1_time_src", base + 0xa780, 28); - clks[IMX7D_ENET2_REF_ROOT_CG] = imx_clk_gate3("enet2_ref_cg", "enet2_ref_src", base + 0xa800, 28); - clks[IMX7D_ENET2_TIME_ROOT_CG] = imx_clk_gate3("enet2_time_cg", "enet2_time_src", base + 0xa880, 28); - clks[IMX7D_ENET_PHY_REF_ROOT_CG] = imx_clk_gate3("enet_phy_ref_cg", "enet_phy_ref_src", base + 0xa900, 28); - clks[IMX7D_EIM_ROOT_CG] = imx_clk_gate3("eim_cg", "eim_src", base + 0xa980, 28); - clks[IMX7D_NAND_ROOT_CG] = imx_clk_gate3("nand_cg", "nand_src", base + 0xaa00, 28); - clks[IMX7D_QSPI_ROOT_CG] = imx_clk_gate3("qspi_cg", "qspi_src", base + 0xaa80, 28); - clks[IMX7D_USDHC1_ROOT_CG] = imx_clk_gate3("usdhc1_cg", "usdhc1_src", base + 0xab00, 28); - clks[IMX7D_USDHC2_ROOT_CG] = imx_clk_gate3("usdhc2_cg", "usdhc2_src", base + 0xab80, 28); - clks[IMX7D_USDHC3_ROOT_CG] = imx_clk_gate3("usdhc3_cg", "usdhc3_src", base + 0xac00, 28); - clks[IMX7D_CAN1_ROOT_CG] = imx_clk_gate3("can1_cg", "can1_src", base + 0xac80, 28); - clks[IMX7D_CAN2_ROOT_CG] = imx_clk_gate3("can2_cg", "can2_src", base + 0xad00, 28); - clks[IMX7D_I2C1_ROOT_CG] = imx_clk_gate3("i2c1_cg", "i2c1_src", base + 0xad80, 28); - clks[IMX7D_I2C2_ROOT_CG] = imx_clk_gate3("i2c2_cg", "i2c2_src", base + 0xae00, 28); - clks[IMX7D_I2C3_ROOT_CG] = imx_clk_gate3("i2c3_cg", "i2c3_src", base + 0xae80, 28); - clks[IMX7D_I2C4_ROOT_CG] = imx_clk_gate3("i2c4_cg", "i2c4_src", base + 0xaf00, 28); - clks[IMX7D_UART1_ROOT_CG] = imx_clk_gate3("uart1_cg", "uart1_src", base + 0xaf80, 28); - clks[IMX7D_UART2_ROOT_CG] = imx_clk_gate3("uart2_cg", "uart2_src", base + 0xb000, 28); - clks[IMX7D_UART3_ROOT_CG] = imx_clk_gate3("uart3_cg", "uart3_src", base + 0xb080, 28); - clks[IMX7D_UART4_ROOT_CG] = imx_clk_gate3("uart4_cg", "uart4_src", base + 0xb100, 28); - clks[IMX7D_UART5_ROOT_CG] = imx_clk_gate3("uart5_cg", "uart5_src", base + 0xb180, 28); - clks[IMX7D_UART6_ROOT_CG] = imx_clk_gate3("uart6_cg", "uart6_src", base + 0xb200, 28); - clks[IMX7D_UART7_ROOT_CG] = imx_clk_gate3("uart7_cg", "uart7_src", base + 0xb280, 28); - clks[IMX7D_ECSPI1_ROOT_CG] = imx_clk_gate3("ecspi1_cg", "ecspi1_src", base + 0xb300, 28); - clks[IMX7D_ECSPI2_ROOT_CG] = imx_clk_gate3("ecspi2_cg", "ecspi2_src", base + 0xb380, 28); - clks[IMX7D_ECSPI3_ROOT_CG] = imx_clk_gate3("ecspi3_cg", "ecspi3_src", base + 0xb400, 28); - clks[IMX7D_ECSPI4_ROOT_CG] = imx_clk_gate3("ecspi4_cg", "ecspi4_src", base + 0xb480, 28); - clks[IMX7D_PWM1_ROOT_CG] = imx_clk_gate3("pwm1_cg", "pwm1_src", base + 0xb500, 28); - clks[IMX7D_PWM2_ROOT_CG] = imx_clk_gate3("pwm2_cg", "pwm2_src", base + 0xb580, 28); - clks[IMX7D_PWM3_ROOT_CG] = imx_clk_gate3("pwm3_cg", "pwm3_src", base + 0xb600, 28); - clks[IMX7D_PWM4_ROOT_CG] = imx_clk_gate3("pwm4_cg", "pwm4_src", base + 0xb680, 28); - clks[IMX7D_FLEXTIMER1_ROOT_CG] = imx_clk_gate3("flextimer1_cg", "flextimer1_src", base + 0xb700, 28); - clks[IMX7D_FLEXTIMER2_ROOT_CG] = imx_clk_gate3("flextimer2_cg", "flextimer2_src", base + 0xb780, 28); - clks[IMX7D_SIM1_ROOT_CG] = imx_clk_gate3("sim1_cg", "sim1_src", base + 0xb800, 28); - clks[IMX7D_SIM2_ROOT_CG] = imx_clk_gate3("sim2_cg", "sim2_src", base + 0xb880, 28); - clks[IMX7D_GPT1_ROOT_CG] = imx_clk_gate3("gpt1_cg", "gpt1_src", base + 0xb900, 28); - clks[IMX7D_GPT2_ROOT_CG] = imx_clk_gate3("gpt2_cg", "gpt2_src", base + 0xb980, 28); - clks[IMX7D_GPT3_ROOT_CG] = imx_clk_gate3("gpt3_cg", "gpt3_src", base + 0xbA00, 28); - clks[IMX7D_GPT4_ROOT_CG] = imx_clk_gate3("gpt4_cg", "gpt4_src", base + 0xbA80, 28); - clks[IMX7D_TRACE_ROOT_CG] = imx_clk_gate3("trace_cg", "trace_src", base + 0xbb00, 28); - clks[IMX7D_WDOG_ROOT_CG] = imx_clk_gate3("wdog_cg", "wdog_src", base + 0xbb80, 28); - clks[IMX7D_CSI_MCLK_ROOT_CG] = imx_clk_gate3("csi_mclk_cg", "csi_mclk_src", base + 0xbc00, 28); - clks[IMX7D_AUDIO_MCLK_ROOT_CG] = imx_clk_gate3("audio_mclk_cg", "audio_mclk_src", base + 0xbc80, 28); - clks[IMX7D_WRCLK_ROOT_CG] = imx_clk_gate3("wrclk_cg", "wrclk_src", base + 0xbd00, 28); - clks[IMX7D_CLKO1_ROOT_CG] = imx_clk_gate3("clko1_cg", "clko1_src", base + 0xbd80, 28); - clks[IMX7D_CLKO2_ROOT_CG] = imx_clk_gate3("clko2_cg", "clko2_src", base + 0xbe00, 28); - - clks[IMX7D_MAIN_AXI_ROOT_PRE_DIV] = imx_clk_divider2("axi_pre_div", "axi_cg", base + 0x8800, 16, 3); - clks[IMX7D_DISP_AXI_ROOT_PRE_DIV] = imx_clk_divider2("disp_axi_pre_div", "disp_axi_cg", base + 0x8880, 16, 3); - clks[IMX7D_ENET_AXI_ROOT_PRE_DIV] = imx_clk_divider2("enet_axi_pre_div", "enet_axi_cg", base + 0x8900, 16, 3); - clks[IMX7D_NAND_USDHC_BUS_ROOT_PRE_DIV] = imx_clk_divider2("nand_usdhc_pre_div", "nand_usdhc_cg", base + 0x8980, 16, 3); - clks[IMX7D_AHB_CHANNEL_ROOT_PRE_DIV] = imx_clk_divider2("ahb_pre_div", "ahb_cg", base + 0x9000, 16, 3); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_PRE_DIV] = imx_clk_divider2("dram_phym_alt_pre_div", "dram_phym_alt_cg", base + 0xa000, 16, 3); - clks[IMX7D_DRAM_ALT_ROOT_PRE_DIV] = imx_clk_divider2("dram_alt_pre_div", "dram_alt_cg", base + 0xa080, 16, 3); - clks[IMX7D_USB_HSIC_ROOT_PRE_DIV] = imx_clk_divider2("usb_hsic_pre_div", "usb_hsic_cg", base + 0xa100, 16, 3); - clks[IMX7D_PCIE_CTRL_ROOT_PRE_DIV] = imx_clk_divider2("pcie_ctrl_pre_div", "pcie_ctrl_cg", base + 0xa180, 16, 3); - clks[IMX7D_PCIE_PHY_ROOT_PRE_DIV] = imx_clk_divider2("pcie_phy_pre_div", "pcie_phy_cg", base + 0xa200, 16, 3); - clks[IMX7D_EPDC_PIXEL_ROOT_PRE_DIV] = imx_clk_divider2("epdc_pixel_pre_div", "epdc_pixel_cg", base + 0xa280, 16, 3); - clks[IMX7D_LCDIF_PIXEL_ROOT_PRE_DIV] = imx_clk_divider2("lcdif_pixel_pre_div", "lcdif_pixel_cg", base + 0xa300, 16, 3); - clks[IMX7D_MIPI_DSI_ROOT_PRE_DIV] = imx_clk_divider2("mipi_dsi_pre_div", "mipi_dsi_cg", base + 0xa380, 16, 3); - clks[IMX7D_MIPI_CSI_ROOT_PRE_DIV] = imx_clk_divider2("mipi_csi_pre_div", "mipi_csi_cg", base + 0xa400, 16, 3); - clks[IMX7D_MIPI_DPHY_ROOT_PRE_DIV] = imx_clk_divider2("mipi_dphy_pre_div", "mipi_dphy_cg", base + 0xa480, 16, 3); - clks[IMX7D_SAI1_ROOT_PRE_DIV] = imx_clk_divider2("sai1_pre_div", "sai1_cg", base + 0xa500, 16, 3); - clks[IMX7D_SAI2_ROOT_PRE_DIV] = imx_clk_divider2("sai2_pre_div", "sai2_cg", base + 0xa580, 16, 3); - clks[IMX7D_SAI3_ROOT_PRE_DIV] = imx_clk_divider2("sai3_pre_div", "sai3_cg", base + 0xa600, 16, 3); - clks[IMX7D_SPDIF_ROOT_PRE_DIV] = imx_clk_divider2("spdif_pre_div", "spdif_cg", base + 0xa680, 16, 3); - clks[IMX7D_ENET1_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet1_ref_pre_div", "enet1_ref_cg", base + 0xa700, 16, 3); - clks[IMX7D_ENET1_TIME_ROOT_PRE_DIV] = imx_clk_divider2("enet1_time_pre_div", "enet1_time_cg", base + 0xa780, 16, 3); - clks[IMX7D_ENET2_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet2_ref_pre_div", "enet2_ref_cg", base + 0xa800, 16, 3); - clks[IMX7D_ENET2_TIME_ROOT_PRE_DIV] = imx_clk_divider2("enet2_time_pre_div", "enet2_time_cg", base + 0xa880, 16, 3); - clks[IMX7D_ENET_PHY_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet_phy_ref_pre_div", "enet_phy_ref_cg", base + 0xa900, 16, 3); - clks[IMX7D_EIM_ROOT_PRE_DIV] = imx_clk_divider2("eim_pre_div", "eim_cg", base + 0xa980, 16, 3); - clks[IMX7D_NAND_ROOT_PRE_DIV] = imx_clk_divider2("nand_pre_div", "nand_cg", base + 0xaa00, 16, 3); - clks[IMX7D_QSPI_ROOT_PRE_DIV] = imx_clk_divider2("qspi_pre_div", "qspi_cg", base + 0xaa80, 16, 3); - clks[IMX7D_USDHC1_ROOT_PRE_DIV] = imx_clk_divider2("usdhc1_pre_div", "usdhc1_cg", base + 0xab00, 16, 3); - clks[IMX7D_USDHC2_ROOT_PRE_DIV] = imx_clk_divider2("usdhc2_pre_div", "usdhc2_cg", base + 0xab80, 16, 3); - clks[IMX7D_USDHC3_ROOT_PRE_DIV] = imx_clk_divider2("usdhc3_pre_div", "usdhc3_cg", base + 0xac00, 16, 3); - clks[IMX7D_CAN1_ROOT_PRE_DIV] = imx_clk_divider2("can1_pre_div", "can1_cg", base + 0xac80, 16, 3); - clks[IMX7D_CAN2_ROOT_PRE_DIV] = imx_clk_divider2("can2_pre_div", "can2_cg", base + 0xad00, 16, 3); - clks[IMX7D_I2C1_ROOT_PRE_DIV] = imx_clk_divider2("i2c1_pre_div", "i2c1_cg", base + 0xad80, 16, 3); - clks[IMX7D_I2C2_ROOT_PRE_DIV] = imx_clk_divider2("i2c2_pre_div", "i2c2_cg", base + 0xae00, 16, 3); - clks[IMX7D_I2C3_ROOT_PRE_DIV] = imx_clk_divider2("i2c3_pre_div", "i2c3_cg", base + 0xae80, 16, 3); - clks[IMX7D_I2C4_ROOT_PRE_DIV] = imx_clk_divider2("i2c4_pre_div", "i2c4_cg", base + 0xaf00, 16, 3); - clks[IMX7D_UART1_ROOT_PRE_DIV] = imx_clk_divider2("uart1_pre_div", "uart1_cg", base + 0xaf80, 16, 3); - clks[IMX7D_UART2_ROOT_PRE_DIV] = imx_clk_divider2("uart2_pre_div", "uart2_cg", base + 0xb000, 16, 3); - clks[IMX7D_UART3_ROOT_PRE_DIV] = imx_clk_divider2("uart3_pre_div", "uart3_cg", base + 0xb080, 16, 3); - clks[IMX7D_UART4_ROOT_PRE_DIV] = imx_clk_divider2("uart4_pre_div", "uart4_cg", base + 0xb100, 16, 3); - clks[IMX7D_UART5_ROOT_PRE_DIV] = imx_clk_divider2("uart5_pre_div", "uart5_cg", base + 0xb180, 16, 3); - clks[IMX7D_UART6_ROOT_PRE_DIV] = imx_clk_divider2("uart6_pre_div", "uart6_cg", base + 0xb200, 16, 3); - clks[IMX7D_UART7_ROOT_PRE_DIV] = imx_clk_divider2("uart7_pre_div", "uart7_cg", base + 0xb280, 16, 3); - clks[IMX7D_ECSPI1_ROOT_PRE_DIV] = imx_clk_divider2("ecspi1_pre_div", "ecspi1_cg", base + 0xb300, 16, 3); - clks[IMX7D_ECSPI2_ROOT_PRE_DIV] = imx_clk_divider2("ecspi2_pre_div", "ecspi2_cg", base + 0xb380, 16, 3); - clks[IMX7D_ECSPI3_ROOT_PRE_DIV] = imx_clk_divider2("ecspi3_pre_div", "ecspi3_cg", base + 0xb400, 16, 3); - clks[IMX7D_ECSPI4_ROOT_PRE_DIV] = imx_clk_divider2("ecspi4_pre_div", "ecspi4_cg", base + 0xb480, 16, 3); - clks[IMX7D_PWM1_ROOT_PRE_DIV] = imx_clk_divider2("pwm1_pre_div", "pwm1_cg", base + 0xb500, 16, 3); - clks[IMX7D_PWM2_ROOT_PRE_DIV] = imx_clk_divider2("pwm2_pre_div", "pwm2_cg", base + 0xb580, 16, 3); - clks[IMX7D_PWM3_ROOT_PRE_DIV] = imx_clk_divider2("pwm3_pre_div", "pwm3_cg", base + 0xb600, 16, 3); - clks[IMX7D_PWM4_ROOT_PRE_DIV] = imx_clk_divider2("pwm4_pre_div", "pwm4_cg", base + 0xb680, 16, 3); - clks[IMX7D_FLEXTIMER1_ROOT_PRE_DIV] = imx_clk_divider2("flextimer1_pre_div", "flextimer1_cg", base + 0xb700, 16, 3); - clks[IMX7D_FLEXTIMER2_ROOT_PRE_DIV] = imx_clk_divider2("flextimer2_pre_div", "flextimer2_cg", base + 0xb780, 16, 3); - clks[IMX7D_SIM1_ROOT_PRE_DIV] = imx_clk_divider2("sim1_pre_div", "sim1_cg", base + 0xb800, 16, 3); - clks[IMX7D_SIM2_ROOT_PRE_DIV] = imx_clk_divider2("sim2_pre_div", "sim2_cg", base + 0xb880, 16, 3); - clks[IMX7D_GPT1_ROOT_PRE_DIV] = imx_clk_divider2("gpt1_pre_div", "gpt1_cg", base + 0xb900, 16, 3); - clks[IMX7D_GPT2_ROOT_PRE_DIV] = imx_clk_divider2("gpt2_pre_div", "gpt2_cg", base + 0xb980, 16, 3); - clks[IMX7D_GPT3_ROOT_PRE_DIV] = imx_clk_divider2("gpt3_pre_div", "gpt3_cg", base + 0xba00, 16, 3); - clks[IMX7D_GPT4_ROOT_PRE_DIV] = imx_clk_divider2("gpt4_pre_div", "gpt4_cg", base + 0xba80, 16, 3); - clks[IMX7D_TRACE_ROOT_PRE_DIV] = imx_clk_divider2("trace_pre_div", "trace_cg", base + 0xbb00, 16, 3); - clks[IMX7D_WDOG_ROOT_PRE_DIV] = imx_clk_divider2("wdog_pre_div", "wdog_cg", base + 0xbb80, 16, 3); - clks[IMX7D_CSI_MCLK_ROOT_PRE_DIV] = imx_clk_divider2("csi_mclk_pre_div", "csi_mclk_cg", base + 0xbc00, 16, 3); - clks[IMX7D_AUDIO_MCLK_ROOT_PRE_DIV] = imx_clk_divider2("audio_mclk_pre_div", "audio_mclk_cg", base + 0xbc80, 16, 3); - clks[IMX7D_WRCLK_ROOT_PRE_DIV] = imx_clk_divider2("wrclk_pre_div", "wrclk_cg", base + 0xbd00, 16, 3); - clks[IMX7D_CLKO1_ROOT_PRE_DIV] = imx_clk_divider2("clko1_pre_div", "clko1_cg", base + 0xbd80, 16, 3); - clks[IMX7D_CLKO2_ROOT_PRE_DIV] = imx_clk_divider2("clko2_pre_div", "clko2_cg", base + 0xbe00, 16, 3); - - clks[IMX7D_ARM_A7_ROOT_DIV] = imx_clk_divider2("arm_a7_div", "arm_a7_cg", base + 0x8000, 0, 3); - clks[IMX7D_ARM_M4_ROOT_DIV] = imx_clk_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); - clks[IMX7D_MAIN_AXI_ROOT_DIV] = imx_clk_divider2("axi_post_div", "axi_pre_div", base + 0x8800, 0, 6); - clks[IMX7D_DISP_AXI_ROOT_DIV] = imx_clk_divider2("disp_axi_post_div", "disp_axi_pre_div", base + 0x8880, 0, 6); - clks[IMX7D_ENET_AXI_ROOT_DIV] = imx_clk_divider2("enet_axi_post_div", "enet_axi_pre_div", base + 0x8900, 0, 6); - clks[IMX7D_NAND_USDHC_BUS_ROOT_CLK] = imx_clk_divider2("nand_usdhc_root_clk", "nand_usdhc_pre_div", base + 0x8980, 0, 6); - clks[IMX7D_AHB_CHANNEL_ROOT_DIV] = imx_clk_divider2("ahb_root_clk", "ahb_pre_div", base + 0x9000, 0, 6); - clks[IMX7D_IPG_ROOT_CLK] = imx_clk_divider_flags("ipg_root_clk", "ahb_root_clk", base + 0x9080, 0, 2, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE | CLK_SET_RATE_PARENT); - clks[IMX7D_DRAM_ROOT_DIV] = imx_clk_divider2("dram_post_div", "dram_cg", base + 0x9880, 0, 3); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_DIV] = imx_clk_divider2("dram_phym_alt_post_div", "dram_phym_alt_pre_div", base + 0xa000, 0, 3); - clks[IMX7D_DRAM_ALT_ROOT_DIV] = imx_clk_divider2("dram_alt_post_div", "dram_alt_pre_div", base + 0xa080, 0, 3); - clks[IMX7D_USB_HSIC_ROOT_DIV] = imx_clk_divider2("usb_hsic_post_div", "usb_hsic_pre_div", base + 0xa100, 0, 6); - clks[IMX7D_PCIE_CTRL_ROOT_DIV] = imx_clk_divider2("pcie_ctrl_post_div", "pcie_ctrl_pre_div", base + 0xa180, 0, 6); - clks[IMX7D_PCIE_PHY_ROOT_DIV] = imx_clk_divider2("pcie_phy_post_div", "pcie_phy_pre_div", base + 0xa200, 0, 6); - clks[IMX7D_EPDC_PIXEL_ROOT_DIV] = imx_clk_divider2("epdc_pixel_post_div", "epdc_pixel_pre_div", base + 0xa280, 0, 6); - clks[IMX7D_LCDIF_PIXEL_ROOT_DIV] = imx_clk_divider2("lcdif_pixel_post_div", "lcdif_pixel_pre_div", base + 0xa300, 0, 6); - clks[IMX7D_MIPI_DSI_ROOT_DIV] = imx_clk_divider2("mipi_dsi_post_div", "mipi_dsi_pre_div", base + 0xa380, 0, 6); - clks[IMX7D_MIPI_CSI_ROOT_DIV] = imx_clk_divider2("mipi_csi_post_div", "mipi_csi_pre_div", base + 0xa400, 0, 6); - clks[IMX7D_MIPI_DPHY_ROOT_DIV] = imx_clk_divider2("mipi_dphy_post_div", "mipi_dphy_pre_div", base + 0xa480, 0, 6); - clks[IMX7D_SAI1_ROOT_DIV] = imx_clk_divider2("sai1_post_div", "sai1_pre_div", base + 0xa500, 0, 6); - clks[IMX7D_SAI2_ROOT_DIV] = imx_clk_divider2("sai2_post_div", "sai2_pre_div", base + 0xa580, 0, 6); - clks[IMX7D_SAI3_ROOT_DIV] = imx_clk_divider2("sai3_post_div", "sai3_pre_div", base + 0xa600, 0, 6); - clks[IMX7D_SPDIF_ROOT_DIV] = imx_clk_divider2("spdif_post_div", "spdif_pre_div", base + 0xa680, 0, 6); - clks[IMX7D_ENET1_REF_ROOT_DIV] = imx_clk_divider2("enet1_ref_post_div", "enet1_ref_pre_div", base + 0xa700, 0, 6); - clks[IMX7D_ENET1_TIME_ROOT_DIV] = imx_clk_divider2("enet1_time_post_div", "enet1_time_pre_div", base + 0xa780, 0, 6); - clks[IMX7D_ENET2_REF_ROOT_DIV] = imx_clk_divider2("enet2_ref_post_div", "enet2_ref_pre_div", base + 0xa800, 0, 6); - clks[IMX7D_ENET2_TIME_ROOT_DIV] = imx_clk_divider2("enet2_time_post_div", "enet2_time_pre_div", base + 0xa880, 0, 6); - clks[IMX7D_ENET_PHY_REF_ROOT_CLK] = imx_clk_divider2("enet_phy_ref_root_clk", "enet_phy_ref_pre_div", base + 0xa900, 0, 6); - clks[IMX7D_EIM_ROOT_DIV] = imx_clk_divider2("eim_post_div", "eim_pre_div", base + 0xa980, 0, 6); - clks[IMX7D_NAND_ROOT_CLK] = imx_clk_divider2("nand_root_clk", "nand_pre_div", base + 0xaa00, 0, 6); - clks[IMX7D_QSPI_ROOT_DIV] = imx_clk_divider2("qspi_post_div", "qspi_pre_div", base + 0xaa80, 0, 6); - clks[IMX7D_USDHC1_ROOT_DIV] = imx_clk_divider2("usdhc1_post_div", "usdhc1_pre_div", base + 0xab00, 0, 6); - clks[IMX7D_USDHC2_ROOT_DIV] = imx_clk_divider2("usdhc2_post_div", "usdhc2_pre_div", base + 0xab80, 0, 6); - clks[IMX7D_USDHC3_ROOT_DIV] = imx_clk_divider2("usdhc3_post_div", "usdhc3_pre_div", base + 0xac00, 0, 6); - clks[IMX7D_CAN1_ROOT_DIV] = imx_clk_divider2("can1_post_div", "can1_pre_div", base + 0xac80, 0, 6); - clks[IMX7D_CAN2_ROOT_DIV] = imx_clk_divider2("can2_post_div", "can2_pre_div", base + 0xad00, 0, 6); - clks[IMX7D_I2C1_ROOT_DIV] = imx_clk_divider2("i2c1_post_div", "i2c1_pre_div", base + 0xad80, 0, 6); - clks[IMX7D_I2C2_ROOT_DIV] = imx_clk_divider2("i2c2_post_div", "i2c2_pre_div", base + 0xae00, 0, 6); - clks[IMX7D_I2C3_ROOT_DIV] = imx_clk_divider2("i2c3_post_div", "i2c3_pre_div", base + 0xae80, 0, 6); - clks[IMX7D_I2C4_ROOT_DIV] = imx_clk_divider2("i2c4_post_div", "i2c4_pre_div", base + 0xaf00, 0, 6); - clks[IMX7D_UART1_ROOT_DIV] = imx_clk_divider2("uart1_post_div", "uart1_pre_div", base + 0xaf80, 0, 6); - clks[IMX7D_UART2_ROOT_DIV] = imx_clk_divider2("uart2_post_div", "uart2_pre_div", base + 0xb000, 0, 6); - clks[IMX7D_UART3_ROOT_DIV] = imx_clk_divider2("uart3_post_div", "uart3_pre_div", base + 0xb080, 0, 6); - clks[IMX7D_UART4_ROOT_DIV] = imx_clk_divider2("uart4_post_div", "uart4_pre_div", base + 0xb100, 0, 6); - clks[IMX7D_UART5_ROOT_DIV] = imx_clk_divider2("uart5_post_div", "uart5_pre_div", base + 0xb180, 0, 6); - clks[IMX7D_UART6_ROOT_DIV] = imx_clk_divider2("uart6_post_div", "uart6_pre_div", base + 0xb200, 0, 6); - clks[IMX7D_UART7_ROOT_DIV] = imx_clk_divider2("uart7_post_div", "uart7_pre_div", base + 0xb280, 0, 6); - clks[IMX7D_ECSPI1_ROOT_DIV] = imx_clk_divider2("ecspi1_post_div", "ecspi1_pre_div", base + 0xb300, 0, 6); - clks[IMX7D_ECSPI2_ROOT_DIV] = imx_clk_divider2("ecspi2_post_div", "ecspi2_pre_div", base + 0xb380, 0, 6); - clks[IMX7D_ECSPI3_ROOT_DIV] = imx_clk_divider2("ecspi3_post_div", "ecspi3_pre_div", base + 0xb400, 0, 6); - clks[IMX7D_ECSPI4_ROOT_DIV] = imx_clk_divider2("ecspi4_post_div", "ecspi4_pre_div", base + 0xb480, 0, 6); - clks[IMX7D_PWM1_ROOT_DIV] = imx_clk_divider2("pwm1_post_div", "pwm1_pre_div", base + 0xb500, 0, 6); - clks[IMX7D_PWM2_ROOT_DIV] = imx_clk_divider2("pwm2_post_div", "pwm2_pre_div", base + 0xb580, 0, 6); - clks[IMX7D_PWM3_ROOT_DIV] = imx_clk_divider2("pwm3_post_div", "pwm3_pre_div", base + 0xb600, 0, 6); - clks[IMX7D_PWM4_ROOT_DIV] = imx_clk_divider2("pwm4_post_div", "pwm4_pre_div", base + 0xb680, 0, 6); - clks[IMX7D_FLEXTIMER1_ROOT_DIV] = imx_clk_divider2("flextimer1_post_div", "flextimer1_pre_div", base + 0xb700, 0, 6); - clks[IMX7D_FLEXTIMER2_ROOT_DIV] = imx_clk_divider2("flextimer2_post_div", "flextimer2_pre_div", base + 0xb780, 0, 6); - clks[IMX7D_SIM1_ROOT_DIV] = imx_clk_divider2("sim1_post_div", "sim1_pre_div", base + 0xb800, 0, 6); - clks[IMX7D_SIM2_ROOT_DIV] = imx_clk_divider2("sim2_post_div", "sim2_pre_div", base + 0xb880, 0, 6); - clks[IMX7D_GPT1_ROOT_DIV] = imx_clk_divider2("gpt1_post_div", "gpt1_pre_div", base + 0xb900, 0, 6); - clks[IMX7D_GPT2_ROOT_DIV] = imx_clk_divider2("gpt2_post_div", "gpt2_pre_div", base + 0xb980, 0, 6); - clks[IMX7D_GPT3_ROOT_DIV] = imx_clk_divider2("gpt3_post_div", "gpt3_pre_div", base + 0xba00, 0, 6); - clks[IMX7D_GPT4_ROOT_DIV] = imx_clk_divider2("gpt4_post_div", "gpt4_pre_div", base + 0xba80, 0, 6); - clks[IMX7D_TRACE_ROOT_DIV] = imx_clk_divider2("trace_post_div", "trace_pre_div", base + 0xbb00, 0, 6); - clks[IMX7D_WDOG_ROOT_DIV] = imx_clk_divider2("wdog_post_div", "wdog_pre_div", base + 0xbb80, 0, 6); - clks[IMX7D_CSI_MCLK_ROOT_DIV] = imx_clk_divider2("csi_mclk_post_div", "csi_mclk_pre_div", base + 0xbc00, 0, 6); - clks[IMX7D_AUDIO_MCLK_ROOT_DIV] = imx_clk_divider2("audio_mclk_post_div", "audio_mclk_pre_div", base + 0xbc80, 0, 6); - clks[IMX7D_WRCLK_ROOT_DIV] = imx_clk_divider2("wrclk_post_div", "wrclk_pre_div", base + 0xbd00, 0, 6); - clks[IMX7D_CLKO1_ROOT_DIV] = imx_clk_divider2("clko1_post_div", "clko1_pre_div", base + 0xbd80, 0, 6); - clks[IMX7D_CLKO2_ROOT_DIV] = imx_clk_divider2("clko2_post_div", "clko2_pre_div", base + 0xbe00, 0, 6); - - clks[IMX7D_ARM_A7_ROOT_CLK] = imx_clk_gate2_flags("arm_a7_root_clk", "arm_a7_div", base + 0x4000, 0, CLK_OPS_PARENT_ENABLE); - clks[IMX7D_ARM_M4_ROOT_CLK] = imx_clk_gate4("arm_m4_root_clk", "arm_m4_div", base + 0x4010, 0); - clks[IMX7D_MAIN_AXI_ROOT_CLK] = imx_clk_gate2_flags("main_axi_root_clk", "axi_post_div", base + 0x4040, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); - clks[IMX7D_DISP_AXI_ROOT_CLK] = imx_clk_gate4("disp_axi_root_clk", "disp_axi_post_div", base + 0x4050, 0); - clks[IMX7D_ENET_AXI_ROOT_CLK] = imx_clk_gate4("enet_axi_root_clk", "enet_axi_post_div", base + 0x4060, 0); - clks[IMX7D_OCRAM_CLK] = imx_clk_gate4("ocram_clk", "main_axi_root_clk", base + 0x4110, 0); - clks[IMX7D_OCRAM_S_CLK] = imx_clk_gate4("ocram_s_clk", "ahb_root_clk", base + 0x4120, 0); - clks[IMX7D_DRAM_ROOT_CLK] = imx_clk_gate2_flags("dram_root_clk", "dram_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); - clks[IMX7D_DRAM_PHYM_ROOT_CLK] = imx_clk_gate2_flags("dram_phym_root_clk", "dram_phym_cg", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_CLK] = imx_clk_gate2_flags("dram_phym_alt_root_clk", "dram_phym_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); - clks[IMX7D_DRAM_ALT_ROOT_CLK] = imx_clk_gate2_flags("dram_alt_root_clk", "dram_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); - clks[IMX7D_OCOTP_CLK] = imx_clk_gate4("ocotp_clk", "ipg_root_clk", base + 0x4230, 0); - clks[IMX7D_SNVS_CLK] = imx_clk_gate4("snvs_clk", "ipg_root_clk", base + 0x4250, 0); - clks[IMX7D_MU_ROOT_CLK] = imx_clk_gate4("mu_root_clk", "ipg_root_clk", base + 0x4270, 0); - clks[IMX7D_CAAM_CLK] = imx_clk_gate4("caam_clk", "ipg_root_clk", base + 0x4240, 0); - clks[IMX7D_USB_HSIC_ROOT_CLK] = imx_clk_gate4("usb_hsic_root_clk", "usb_hsic_post_div", base + 0x4690, 0); - clks[IMX7D_SDMA_CORE_CLK] = imx_clk_gate4("sdma_root_clk", "ahb_root_clk", base + 0x4480, 0); - clks[IMX7D_PCIE_CTRL_ROOT_CLK] = imx_clk_gate4("pcie_ctrl_root_clk", "pcie_ctrl_post_div", base + 0x4600, 0); - clks[IMX7D_PCIE_PHY_ROOT_CLK] = imx_clk_gate4("pcie_phy_root_clk", "pcie_phy_post_div", base + 0x4600, 0); - clks[IMX7D_EPDC_PIXEL_ROOT_CLK] = imx_clk_gate4("epdc_pixel_root_clk", "epdc_pixel_post_div", base + 0x44a0, 0); - clks[IMX7D_LCDIF_PIXEL_ROOT_CLK] = imx_clk_gate4("lcdif_pixel_root_clk", "lcdif_pixel_post_div", base + 0x44b0, 0); - clks[IMX7D_MIPI_DSI_ROOT_CLK] = imx_clk_gate4("mipi_dsi_root_clk", "mipi_dsi_post_div", base + 0x4650, 0); - clks[IMX7D_MIPI_CSI_ROOT_CLK] = imx_clk_gate4("mipi_csi_root_clk", "mipi_csi_post_div", base + 0x4640, 0); - clks[IMX7D_MIPI_DPHY_ROOT_CLK] = imx_clk_gate4("mipi_dphy_root_clk", "mipi_dphy_post_div", base + 0x4660, 0); - clks[IMX7D_ENET1_IPG_ROOT_CLK] = imx_clk_gate2_shared2("enet1_ipg_root_clk", "enet_axi_post_div", base + 0x4700, 0, &share_count_enet1); - clks[IMX7D_ENET1_TIME_ROOT_CLK] = imx_clk_gate2_shared2("enet1_time_root_clk", "enet1_time_post_div", base + 0x4700, 0, &share_count_enet1); - clks[IMX7D_ENET2_IPG_ROOT_CLK] = imx_clk_gate2_shared2("enet2_ipg_root_clk", "enet_axi_post_div", base + 0x4710, 0, &share_count_enet2); - clks[IMX7D_ENET2_TIME_ROOT_CLK] = imx_clk_gate2_shared2("enet2_time_root_clk", "enet2_time_post_div", base + 0x4710, 0, &share_count_enet2); - clks[IMX7D_SAI1_ROOT_CLK] = imx_clk_gate2_shared2("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0, &share_count_sai1); - clks[IMX7D_SAI1_IPG_CLK] = imx_clk_gate2_shared2("sai1_ipg_clk", "ipg_root_clk", base + 0x48c0, 0, &share_count_sai1); - clks[IMX7D_SAI2_ROOT_CLK] = imx_clk_gate2_shared2("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0, &share_count_sai2); - clks[IMX7D_SAI2_IPG_CLK] = imx_clk_gate2_shared2("sai2_ipg_clk", "ipg_root_clk", base + 0x48d0, 0, &share_count_sai2); - clks[IMX7D_SAI3_ROOT_CLK] = imx_clk_gate2_shared2("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0, &share_count_sai3); - clks[IMX7D_SAI3_IPG_CLK] = imx_clk_gate2_shared2("sai3_ipg_clk", "ipg_root_clk", base + 0x48e0, 0, &share_count_sai3); - clks[IMX7D_SPDIF_ROOT_CLK] = imx_clk_gate4("spdif_root_clk", "spdif_post_div", base + 0x44d0, 0); - clks[IMX7D_EIM_ROOT_CLK] = imx_clk_gate4("eim_root_clk", "eim_post_div", base + 0x4160, 0); - clks[IMX7D_NAND_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_rawnand_clk", "nand_root_clk", base + 0x4140, 0, &share_count_nand); - clks[IMX7D_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_root_clk", base + 0x4140, 0, &share_count_nand); - clks[IMX7D_QSPI_ROOT_CLK] = imx_clk_gate4("qspi_root_clk", "qspi_post_div", base + 0x4150, 0); - clks[IMX7D_USDHC1_ROOT_CLK] = imx_clk_gate4("usdhc1_root_clk", "usdhc1_post_div", base + 0x46c0, 0); - clks[IMX7D_USDHC2_ROOT_CLK] = imx_clk_gate4("usdhc2_root_clk", "usdhc2_post_div", base + 0x46d0, 0); - clks[IMX7D_USDHC3_ROOT_CLK] = imx_clk_gate4("usdhc3_root_clk", "usdhc3_post_div", base + 0x46e0, 0); - clks[IMX7D_CAN1_ROOT_CLK] = imx_clk_gate4("can1_root_clk", "can1_post_div", base + 0x4740, 0); - clks[IMX7D_CAN2_ROOT_CLK] = imx_clk_gate4("can2_root_clk", "can2_post_div", base + 0x4750, 0); - clks[IMX7D_I2C1_ROOT_CLK] = imx_clk_gate4("i2c1_root_clk", "i2c1_post_div", base + 0x4880, 0); - clks[IMX7D_I2C2_ROOT_CLK] = imx_clk_gate4("i2c2_root_clk", "i2c2_post_div", base + 0x4890, 0); - clks[IMX7D_I2C3_ROOT_CLK] = imx_clk_gate4("i2c3_root_clk", "i2c3_post_div", base + 0x48a0, 0); - clks[IMX7D_I2C4_ROOT_CLK] = imx_clk_gate4("i2c4_root_clk", "i2c4_post_div", base + 0x48b0, 0); - clks[IMX7D_UART1_ROOT_CLK] = imx_clk_gate4("uart1_root_clk", "uart1_post_div", base + 0x4940, 0); - clks[IMX7D_UART2_ROOT_CLK] = imx_clk_gate4("uart2_root_clk", "uart2_post_div", base + 0x4950, 0); - clks[IMX7D_UART3_ROOT_CLK] = imx_clk_gate4("uart3_root_clk", "uart3_post_div", base + 0x4960, 0); - clks[IMX7D_UART4_ROOT_CLK] = imx_clk_gate4("uart4_root_clk", "uart4_post_div", base + 0x4970, 0); - clks[IMX7D_UART5_ROOT_CLK] = imx_clk_gate4("uart5_root_clk", "uart5_post_div", base + 0x4980, 0); - clks[IMX7D_UART6_ROOT_CLK] = imx_clk_gate4("uart6_root_clk", "uart6_post_div", base + 0x4990, 0); - clks[IMX7D_UART7_ROOT_CLK] = imx_clk_gate4("uart7_root_clk", "uart7_post_div", base + 0x49a0, 0); - clks[IMX7D_ECSPI1_ROOT_CLK] = imx_clk_gate4("ecspi1_root_clk", "ecspi1_post_div", base + 0x4780, 0); - clks[IMX7D_ECSPI2_ROOT_CLK] = imx_clk_gate4("ecspi2_root_clk", "ecspi2_post_div", base + 0x4790, 0); - clks[IMX7D_ECSPI3_ROOT_CLK] = imx_clk_gate4("ecspi3_root_clk", "ecspi3_post_div", base + 0x47a0, 0); - clks[IMX7D_ECSPI4_ROOT_CLK] = imx_clk_gate4("ecspi4_root_clk", "ecspi4_post_div", base + 0x47b0, 0); - clks[IMX7D_PWM1_ROOT_CLK] = imx_clk_gate4("pwm1_root_clk", "pwm1_post_div", base + 0x4840, 0); - clks[IMX7D_PWM2_ROOT_CLK] = imx_clk_gate4("pwm2_root_clk", "pwm2_post_div", base + 0x4850, 0); - clks[IMX7D_PWM3_ROOT_CLK] = imx_clk_gate4("pwm3_root_clk", "pwm3_post_div", base + 0x4860, 0); - clks[IMX7D_PWM4_ROOT_CLK] = imx_clk_gate4("pwm4_root_clk", "pwm4_post_div", base + 0x4870, 0); - clks[IMX7D_FLEXTIMER1_ROOT_CLK] = imx_clk_gate4("flextimer1_root_clk", "flextimer1_post_div", base + 0x4800, 0); - clks[IMX7D_FLEXTIMER2_ROOT_CLK] = imx_clk_gate4("flextimer2_root_clk", "flextimer2_post_div", base + 0x4810, 0); - clks[IMX7D_SIM1_ROOT_CLK] = imx_clk_gate4("sim1_root_clk", "sim1_post_div", base + 0x4900, 0); - clks[IMX7D_SIM2_ROOT_CLK] = imx_clk_gate4("sim2_root_clk", "sim2_post_div", base + 0x4910, 0); - clks[IMX7D_GPT1_ROOT_CLK] = imx_clk_gate4("gpt1_root_clk", "gpt1_post_div", base + 0x47c0, 0); - clks[IMX7D_GPT2_ROOT_CLK] = imx_clk_gate4("gpt2_root_clk", "gpt2_post_div", base + 0x47d0, 0); - clks[IMX7D_GPT3_ROOT_CLK] = imx_clk_gate4("gpt3_root_clk", "gpt3_post_div", base + 0x47e0, 0); - clks[IMX7D_GPT4_ROOT_CLK] = imx_clk_gate4("gpt4_root_clk", "gpt4_post_div", base + 0x47f0, 0); - clks[IMX7D_TRACE_ROOT_CLK] = imx_clk_gate4("trace_root_clk", "trace_post_div", base + 0x4300, 0); - clks[IMX7D_WDOG1_ROOT_CLK] = imx_clk_gate4("wdog1_root_clk", "wdog_post_div", base + 0x49c0, 0); - clks[IMX7D_WDOG2_ROOT_CLK] = imx_clk_gate4("wdog2_root_clk", "wdog_post_div", base + 0x49d0, 0); - clks[IMX7D_WDOG3_ROOT_CLK] = imx_clk_gate4("wdog3_root_clk", "wdog_post_div", base + 0x49e0, 0); - clks[IMX7D_WDOG4_ROOT_CLK] = imx_clk_gate4("wdog4_root_clk", "wdog_post_div", base + 0x49f0, 0); - clks[IMX7D_KPP_ROOT_CLK] = imx_clk_gate4("kpp_root_clk", "ipg_root_clk", base + 0x4aa0, 0); - clks[IMX7D_CSI_MCLK_ROOT_CLK] = imx_clk_gate4("csi_mclk_root_clk", "csi_mclk_post_div", base + 0x4490, 0); - clks[IMX7D_AUDIO_MCLK_ROOT_CLK] = imx_clk_gate4("audio_mclk_root_clk", "audio_mclk_post_div", base + 0x4790, 0); - clks[IMX7D_WRCLK_ROOT_CLK] = imx_clk_gate4("wrclk_root_clk", "wrclk_post_div", base + 0x47a0, 0); - clks[IMX7D_USB_CTRL_CLK] = imx_clk_gate4("usb_ctrl_clk", "ahb_root_clk", base + 0x4680, 0); - clks[IMX7D_USB_PHY1_CLK] = imx_clk_gate4("usb_phy1_clk", "pll_usb1_main_clk", base + 0x46a0, 0); - clks[IMX7D_USB_PHY2_CLK] = imx_clk_gate4("usb_phy2_clk", "pll_usb_main_clk", base + 0x46b0, 0); - clks[IMX7D_ADC_ROOT_CLK] = imx_clk_gate4("adc_root_clk", "ipg_root_clk", base + 0x4200, 0); - - clks[IMX7D_GPT_3M_CLK] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8); - - clks[IMX7D_CLK_ARM] = imx_clk_cpu("arm", "arm_a7_root_clk", - clks[IMX7D_ARM_A7_ROOT_CLK], - clks[IMX7D_ARM_A7_ROOT_SRC], - clks[IMX7D_PLL_ARM_MAIN_CLK], - clks[IMX7D_PLL_SYS_MAIN_CLK]); - - imx_check_clocks(clks, ARRAY_SIZE(clks)); - - clk_data.clks = clks; - clk_data.clk_num = ARRAY_SIZE(clks); - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); - - clk_set_parent(clks[IMX7D_PLL_ARM_MAIN_BYPASS], clks[IMX7D_PLL_ARM_MAIN]); - clk_set_parent(clks[IMX7D_PLL_DRAM_MAIN_BYPASS], clks[IMX7D_PLL_DRAM_MAIN]); - clk_set_parent(clks[IMX7D_PLL_SYS_MAIN_BYPASS], clks[IMX7D_PLL_SYS_MAIN]); - clk_set_parent(clks[IMX7D_PLL_ENET_MAIN_BYPASS], clks[IMX7D_PLL_ENET_MAIN]); - clk_set_parent(clks[IMX7D_PLL_AUDIO_MAIN_BYPASS], clks[IMX7D_PLL_AUDIO_MAIN]); - clk_set_parent(clks[IMX7D_PLL_VIDEO_MAIN_BYPASS], clks[IMX7D_PLL_VIDEO_MAIN]); - - clk_set_parent(clks[IMX7D_MIPI_CSI_ROOT_SRC], clks[IMX7D_PLL_SYS_PFD3_CLK]); + hws[IMX7D_ARM_A7_ROOT_SRC] = imx_clk_hw_mux2("arm_a7_src", base + 0x8000, 24, 3, arm_a7_sel, ARRAY_SIZE(arm_a7_sel)); + hws[IMX7D_ARM_M4_ROOT_SRC] = imx_clk_hw_mux2("arm_m4_src", base + 0x8080, 24, 3, arm_m4_sel, ARRAY_SIZE(arm_m4_sel)); + hws[IMX7D_MAIN_AXI_ROOT_SRC] = imx_clk_hw_mux2("axi_src", base + 0x8800, 24, 3, axi_sel, ARRAY_SIZE(axi_sel)); + hws[IMX7D_DISP_AXI_ROOT_SRC] = imx_clk_hw_mux2("disp_axi_src", base + 0x8880, 24, 3, disp_axi_sel, ARRAY_SIZE(disp_axi_sel)); + hws[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_hw_mux2("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel)); + hws[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_hw_mux2("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel)); + hws[IMX7D_AHB_CHANNEL_ROOT_SRC] = imx_clk_hw_mux2("ahb_src", base + 0x9000, 24, 3, ahb_channel_sel, ARRAY_SIZE(ahb_channel_sel)); + hws[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_hw_mux2("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel)); + hws[IMX7D_DRAM_ROOT_SRC] = imx_clk_hw_mux2("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel)); + hws[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_hw_mux2("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel)); + hws[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_hw_mux2("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel)); + hws[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_hw_mux2("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel)); + hws[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_hw_mux2("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel)); + hws[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_hw_mux2("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel)); + hws[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_hw_mux2("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel)); + hws[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_hw_mux2("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel)); + hws[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_hw_mux2("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel)); + hws[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_hw_mux2("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel)); + hws[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_hw_mux2("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel)); + hws[IMX7D_SAI1_ROOT_SRC] = imx_clk_hw_mux2("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel)); + hws[IMX7D_SAI2_ROOT_SRC] = imx_clk_hw_mux2("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel)); + hws[IMX7D_SAI3_ROOT_SRC] = imx_clk_hw_mux2("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel)); + hws[IMX7D_SPDIF_ROOT_SRC] = imx_clk_hw_mux2("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel)); + hws[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_hw_mux2("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel)); + hws[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_hw_mux2("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel)); + hws[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_hw_mux2("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel)); + hws[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_hw_mux2("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel)); + hws[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_hw_mux2("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel)); + hws[IMX7D_EIM_ROOT_SRC] = imx_clk_hw_mux2("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel)); + hws[IMX7D_NAND_ROOT_SRC] = imx_clk_hw_mux2("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel)); + hws[IMX7D_QSPI_ROOT_SRC] = imx_clk_hw_mux2("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel)); + hws[IMX7D_USDHC1_ROOT_SRC] = imx_clk_hw_mux2("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel)); + hws[IMX7D_USDHC2_ROOT_SRC] = imx_clk_hw_mux2("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel)); + hws[IMX7D_USDHC3_ROOT_SRC] = imx_clk_hw_mux2("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel)); + hws[IMX7D_CAN1_ROOT_SRC] = imx_clk_hw_mux2("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel)); + hws[IMX7D_CAN2_ROOT_SRC] = imx_clk_hw_mux2("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel)); + hws[IMX7D_I2C1_ROOT_SRC] = imx_clk_hw_mux2("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel)); + hws[IMX7D_I2C2_ROOT_SRC] = imx_clk_hw_mux2("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel)); + hws[IMX7D_I2C3_ROOT_SRC] = imx_clk_hw_mux2("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel)); + hws[IMX7D_I2C4_ROOT_SRC] = imx_clk_hw_mux2("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel)); + hws[IMX7D_UART1_ROOT_SRC] = imx_clk_hw_mux2("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel)); + hws[IMX7D_UART2_ROOT_SRC] = imx_clk_hw_mux2("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel)); + hws[IMX7D_UART3_ROOT_SRC] = imx_clk_hw_mux2("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel)); + hws[IMX7D_UART4_ROOT_SRC] = imx_clk_hw_mux2("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel)); + hws[IMX7D_UART5_ROOT_SRC] = imx_clk_hw_mux2("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel)); + hws[IMX7D_UART6_ROOT_SRC] = imx_clk_hw_mux2("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel)); + hws[IMX7D_UART7_ROOT_SRC] = imx_clk_hw_mux2("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel)); + hws[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_hw_mux2("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel)); + hws[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_hw_mux2("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel)); + hws[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_hw_mux2("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel)); + hws[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_hw_mux2("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel)); + hws[IMX7D_PWM1_ROOT_SRC] = imx_clk_hw_mux2("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel)); + hws[IMX7D_PWM2_ROOT_SRC] = imx_clk_hw_mux2("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel)); + hws[IMX7D_PWM3_ROOT_SRC] = imx_clk_hw_mux2("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel)); + hws[IMX7D_PWM4_ROOT_SRC] = imx_clk_hw_mux2("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel)); + hws[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_hw_mux2("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel)); + hws[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_hw_mux2("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel)); + hws[IMX7D_SIM1_ROOT_SRC] = imx_clk_hw_mux2("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel)); + hws[IMX7D_SIM2_ROOT_SRC] = imx_clk_hw_mux2("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel)); + hws[IMX7D_GPT1_ROOT_SRC] = imx_clk_hw_mux2("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel)); + hws[IMX7D_GPT2_ROOT_SRC] = imx_clk_hw_mux2("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel)); + hws[IMX7D_GPT3_ROOT_SRC] = imx_clk_hw_mux2("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel)); + hws[IMX7D_GPT4_ROOT_SRC] = imx_clk_hw_mux2("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel)); + hws[IMX7D_TRACE_ROOT_SRC] = imx_clk_hw_mux2("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel)); + hws[IMX7D_WDOG_ROOT_SRC] = imx_clk_hw_mux2("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel)); + hws[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_hw_mux2("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel)); + hws[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_hw_mux2("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel)); + hws[IMX7D_WRCLK_ROOT_SRC] = imx_clk_hw_mux2("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel)); + hws[IMX7D_CLKO1_ROOT_SRC] = imx_clk_hw_mux2("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel)); + hws[IMX7D_CLKO2_ROOT_SRC] = imx_clk_hw_mux2("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel)); + + hws[IMX7D_ARM_A7_ROOT_CG] = imx_clk_hw_gate3("arm_a7_cg", "arm_a7_src", base + 0x8000, 28); + hws[IMX7D_ARM_M4_ROOT_CG] = imx_clk_hw_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); + hws[IMX7D_MAIN_AXI_ROOT_CG] = imx_clk_hw_gate3("axi_cg", "axi_src", base + 0x8800, 28); + hws[IMX7D_DISP_AXI_ROOT_CG] = imx_clk_hw_gate3("disp_axi_cg", "disp_axi_src", base + 0x8880, 28); + hws[IMX7D_ENET_AXI_ROOT_CG] = imx_clk_hw_gate3("enet_axi_cg", "enet_axi_src", base + 0x8900, 28); + hws[IMX7D_NAND_USDHC_BUS_ROOT_CG] = imx_clk_hw_gate3("nand_usdhc_cg", "nand_usdhc_src", base + 0x8980, 28); + hws[IMX7D_AHB_CHANNEL_ROOT_CG] = imx_clk_hw_gate3("ahb_cg", "ahb_src", base + 0x9000, 28); + hws[IMX7D_DRAM_PHYM_ROOT_CG] = imx_clk_hw_gate3("dram_phym_cg", "dram_phym_src", base + 0x9800, 28); + hws[IMX7D_DRAM_ROOT_CG] = imx_clk_hw_gate3("dram_cg", "dram_src", base + 0x9880, 28); + hws[IMX7D_DRAM_PHYM_ALT_ROOT_CG] = imx_clk_hw_gate3("dram_phym_alt_cg", "dram_phym_alt_src", base + 0xa000, 28); + hws[IMX7D_DRAM_ALT_ROOT_CG] = imx_clk_hw_gate3("dram_alt_cg", "dram_alt_src", base + 0xa080, 28); + hws[IMX7D_USB_HSIC_ROOT_CG] = imx_clk_hw_gate3("usb_hsic_cg", "usb_hsic_src", base + 0xa100, 28); + hws[IMX7D_PCIE_CTRL_ROOT_CG] = imx_clk_hw_gate3("pcie_ctrl_cg", "pcie_ctrl_src", base + 0xa180, 28); + hws[IMX7D_PCIE_PHY_ROOT_CG] = imx_clk_hw_gate3("pcie_phy_cg", "pcie_phy_src", base + 0xa200, 28); + hws[IMX7D_EPDC_PIXEL_ROOT_CG] = imx_clk_hw_gate3("epdc_pixel_cg", "epdc_pixel_src", base + 0xa280, 28); + hws[IMX7D_LCDIF_PIXEL_ROOT_CG] = imx_clk_hw_gate3("lcdif_pixel_cg", "lcdif_pixel_src", base + 0xa300, 28); + hws[IMX7D_MIPI_DSI_ROOT_CG] = imx_clk_hw_gate3("mipi_dsi_cg", "mipi_dsi_src", base + 0xa380, 28); + hws[IMX7D_MIPI_CSI_ROOT_CG] = imx_clk_hw_gate3("mipi_csi_cg", "mipi_csi_src", base + 0xa400, 28); + hws[IMX7D_MIPI_DPHY_ROOT_CG] = imx_clk_hw_gate3("mipi_dphy_cg", "mipi_dphy_src", base + 0xa480, 28); + hws[IMX7D_SAI1_ROOT_CG] = imx_clk_hw_gate3("sai1_cg", "sai1_src", base + 0xa500, 28); + hws[IMX7D_SAI2_ROOT_CG] = imx_clk_hw_gate3("sai2_cg", "sai2_src", base + 0xa580, 28); + hws[IMX7D_SAI3_ROOT_CG] = imx_clk_hw_gate3("sai3_cg", "sai3_src", base + 0xa600, 28); + hws[IMX7D_SPDIF_ROOT_CG] = imx_clk_hw_gate3("spdif_cg", "spdif_src", base + 0xa680, 28); + hws[IMX7D_ENET1_REF_ROOT_CG] = imx_clk_hw_gate3("enet1_ref_cg", "enet1_ref_src", base + 0xa700, 28); + hws[IMX7D_ENET1_TIME_ROOT_CG] = imx_clk_hw_gate3("enet1_time_cg", "enet1_time_src", base + 0xa780, 28); + hws[IMX7D_ENET2_REF_ROOT_CG] = imx_clk_hw_gate3("enet2_ref_cg", "enet2_ref_src", base + 0xa800, 28); + hws[IMX7D_ENET2_TIME_ROOT_CG] = imx_clk_hw_gate3("enet2_time_cg", "enet2_time_src", base + 0xa880, 28); + hws[IMX7D_ENET_PHY_REF_ROOT_CG] = imx_clk_hw_gate3("enet_phy_ref_cg", "enet_phy_ref_src", base + 0xa900, 28); + hws[IMX7D_EIM_ROOT_CG] = imx_clk_hw_gate3("eim_cg", "eim_src", base + 0xa980, 28); + hws[IMX7D_NAND_ROOT_CG] = imx_clk_hw_gate3("nand_cg", "nand_src", base + 0xaa00, 28); + hws[IMX7D_QSPI_ROOT_CG] = imx_clk_hw_gate3("qspi_cg", "qspi_src", base + 0xaa80, 28); + hws[IMX7D_USDHC1_ROOT_CG] = imx_clk_hw_gate3("usdhc1_cg", "usdhc1_src", base + 0xab00, 28); + hws[IMX7D_USDHC2_ROOT_CG] = imx_clk_hw_gate3("usdhc2_cg", "usdhc2_src", base + 0xab80, 28); + hws[IMX7D_USDHC3_ROOT_CG] = imx_clk_hw_gate3("usdhc3_cg", "usdhc3_src", base + 0xac00, 28); + hws[IMX7D_CAN1_ROOT_CG] = imx_clk_hw_gate3("can1_cg", "can1_src", base + 0xac80, 28); + hws[IMX7D_CAN2_ROOT_CG] = imx_clk_hw_gate3("can2_cg", "can2_src", base + 0xad00, 28); + hws[IMX7D_I2C1_ROOT_CG] = imx_clk_hw_gate3("i2c1_cg", "i2c1_src", base + 0xad80, 28); + hws[IMX7D_I2C2_ROOT_CG] = imx_clk_hw_gate3("i2c2_cg", "i2c2_src", base + 0xae00, 28); + hws[IMX7D_I2C3_ROOT_CG] = imx_clk_hw_gate3("i2c3_cg", "i2c3_src", base + 0xae80, 28); + hws[IMX7D_I2C4_ROOT_CG] = imx_clk_hw_gate3("i2c4_cg", "i2c4_src", base + 0xaf00, 28); + hws[IMX7D_UART1_ROOT_CG] = imx_clk_hw_gate3("uart1_cg", "uart1_src", base + 0xaf80, 28); + hws[IMX7D_UART2_ROOT_CG] = imx_clk_hw_gate3("uart2_cg", "uart2_src", base + 0xb000, 28); + hws[IMX7D_UART3_ROOT_CG] = imx_clk_hw_gate3("uart3_cg", "uart3_src", base + 0xb080, 28); + hws[IMX7D_UART4_ROOT_CG] = imx_clk_hw_gate3("uart4_cg", "uart4_src", base + 0xb100, 28); + hws[IMX7D_UART5_ROOT_CG] = imx_clk_hw_gate3("uart5_cg", "uart5_src", base + 0xb180, 28); + hws[IMX7D_UART6_ROOT_CG] = imx_clk_hw_gate3("uart6_cg", "uart6_src", base + 0xb200, 28); + hws[IMX7D_UART7_ROOT_CG] = imx_clk_hw_gate3("uart7_cg", "uart7_src", base + 0xb280, 28); + hws[IMX7D_ECSPI1_ROOT_CG] = imx_clk_hw_gate3("ecspi1_cg", "ecspi1_src", base + 0xb300, 28); + hws[IMX7D_ECSPI2_ROOT_CG] = imx_clk_hw_gate3("ecspi2_cg", "ecspi2_src", base + 0xb380, 28); + hws[IMX7D_ECSPI3_ROOT_CG] = imx_clk_hw_gate3("ecspi3_cg", "ecspi3_src", base + 0xb400, 28); + hws[IMX7D_ECSPI4_ROOT_CG] = imx_clk_hw_gate3("ecspi4_cg", "ecspi4_src", base + 0xb480, 28); + hws[IMX7D_PWM1_ROOT_CG] = imx_clk_hw_gate3("pwm1_cg", "pwm1_src", base + 0xb500, 28); + hws[IMX7D_PWM2_ROOT_CG] = imx_clk_hw_gate3("pwm2_cg", "pwm2_src", base + 0xb580, 28); + hws[IMX7D_PWM3_ROOT_CG] = imx_clk_hw_gate3("pwm3_cg", "pwm3_src", base + 0xb600, 28); + hws[IMX7D_PWM4_ROOT_CG] = imx_clk_hw_gate3("pwm4_cg", "pwm4_src", base + 0xb680, 28); + hws[IMX7D_FLEXTIMER1_ROOT_CG] = imx_clk_hw_gate3("flextimer1_cg", "flextimer1_src", base + 0xb700, 28); + hws[IMX7D_FLEXTIMER2_ROOT_CG] = imx_clk_hw_gate3("flextimer2_cg", "flextimer2_src", base + 0xb780, 28); + hws[IMX7D_SIM1_ROOT_CG] = imx_clk_hw_gate3("sim1_cg", "sim1_src", base + 0xb800, 28); + hws[IMX7D_SIM2_ROOT_CG] = imx_clk_hw_gate3("sim2_cg", "sim2_src", base + 0xb880, 28); + hws[IMX7D_GPT1_ROOT_CG] = imx_clk_hw_gate3("gpt1_cg", "gpt1_src", base + 0xb900, 28); + hws[IMX7D_GPT2_ROOT_CG] = imx_clk_hw_gate3("gpt2_cg", "gpt2_src", base + 0xb980, 28); + hws[IMX7D_GPT3_ROOT_CG] = imx_clk_hw_gate3("gpt3_cg", "gpt3_src", base + 0xbA00, 28); + hws[IMX7D_GPT4_ROOT_CG] = imx_clk_hw_gate3("gpt4_cg", "gpt4_src", base + 0xbA80, 28); + hws[IMX7D_TRACE_ROOT_CG] = imx_clk_hw_gate3("trace_cg", "trace_src", base + 0xbb00, 28); + hws[IMX7D_WDOG_ROOT_CG] = imx_clk_hw_gate3("wdog_cg", "wdog_src", base + 0xbb80, 28); + hws[IMX7D_CSI_MCLK_ROOT_CG] = imx_clk_hw_gate3("csi_mclk_cg", "csi_mclk_src", base + 0xbc00, 28); + hws[IMX7D_AUDIO_MCLK_ROOT_CG] = imx_clk_hw_gate3("audio_mclk_cg", "audio_mclk_src", base + 0xbc80, 28); + hws[IMX7D_WRCLK_ROOT_CG] = imx_clk_hw_gate3("wrclk_cg", "wrclk_src", base + 0xbd00, 28); + hws[IMX7D_CLKO1_ROOT_CG] = imx_clk_hw_gate3("clko1_cg", "clko1_src", base + 0xbd80, 28); + hws[IMX7D_CLKO2_ROOT_CG] = imx_clk_hw_gate3("clko2_cg", "clko2_src", base + 0xbe00, 28); + + hws[IMX7D_MAIN_AXI_ROOT_PRE_DIV] = imx_clk_hw_divider2("axi_pre_div", "axi_cg", base + 0x8800, 16, 3); + hws[IMX7D_DISP_AXI_ROOT_PRE_DIV] = imx_clk_hw_divider2("disp_axi_pre_div", "disp_axi_cg", base + 0x8880, 16, 3); + hws[IMX7D_ENET_AXI_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet_axi_pre_div", "enet_axi_cg", base + 0x8900, 16, 3); + hws[IMX7D_NAND_USDHC_BUS_ROOT_PRE_DIV] = imx_clk_hw_divider2("nand_usdhc_pre_div", "nand_usdhc_cg", base + 0x8980, 16, 3); + hws[IMX7D_AHB_CHANNEL_ROOT_PRE_DIV] = imx_clk_hw_divider2("ahb_pre_div", "ahb_cg", base + 0x9000, 16, 3); + hws[IMX7D_DRAM_PHYM_ALT_ROOT_PRE_DIV] = imx_clk_hw_divider2("dram_phym_alt_pre_div", "dram_phym_alt_cg", base + 0xa000, 16, 3); + hws[IMX7D_DRAM_ALT_ROOT_PRE_DIV] = imx_clk_hw_divider2("dram_alt_pre_div", "dram_alt_cg", base + 0xa080, 16, 3); + hws[IMX7D_USB_HSIC_ROOT_PRE_DIV] = imx_clk_hw_divider2("usb_hsic_pre_div", "usb_hsic_cg", base + 0xa100, 16, 3); + hws[IMX7D_PCIE_CTRL_ROOT_PRE_DIV] = imx_clk_hw_divider2("pcie_ctrl_pre_div", "pcie_ctrl_cg", base + 0xa180, 16, 3); + hws[IMX7D_PCIE_PHY_ROOT_PRE_DIV] = imx_clk_hw_divider2("pcie_phy_pre_div", "pcie_phy_cg", base + 0xa200, 16, 3); + hws[IMX7D_EPDC_PIXEL_ROOT_PRE_DIV] = imx_clk_hw_divider2("epdc_pixel_pre_div", "epdc_pixel_cg", base + 0xa280, 16, 3); + hws[IMX7D_LCDIF_PIXEL_ROOT_PRE_DIV] = imx_clk_hw_divider2("lcdif_pixel_pre_div", "lcdif_pixel_cg", base + 0xa300, 16, 3); + hws[IMX7D_MIPI_DSI_ROOT_PRE_DIV] = imx_clk_hw_divider2("mipi_dsi_pre_div", "mipi_dsi_cg", base + 0xa380, 16, 3); + hws[IMX7D_MIPI_CSI_ROOT_PRE_DIV] = imx_clk_hw_divider2("mipi_csi_pre_div", "mipi_csi_cg", base + 0xa400, 16, 3); + hws[IMX7D_MIPI_DPHY_ROOT_PRE_DIV] = imx_clk_hw_divider2("mipi_dphy_pre_div", "mipi_dphy_cg", base + 0xa480, 16, 3); + hws[IMX7D_SAI1_ROOT_PRE_DIV] = imx_clk_hw_divider2("sai1_pre_div", "sai1_cg", base + 0xa500, 16, 3); + hws[IMX7D_SAI2_ROOT_PRE_DIV] = imx_clk_hw_divider2("sai2_pre_div", "sai2_cg", base + 0xa580, 16, 3); + hws[IMX7D_SAI3_ROOT_PRE_DIV] = imx_clk_hw_divider2("sai3_pre_div", "sai3_cg", base + 0xa600, 16, 3); + hws[IMX7D_SPDIF_ROOT_PRE_DIV] = imx_clk_hw_divider2("spdif_pre_div", "spdif_cg", base + 0xa680, 16, 3); + hws[IMX7D_ENET1_REF_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet1_ref_pre_div", "enet1_ref_cg", base + 0xa700, 16, 3); + hws[IMX7D_ENET1_TIME_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet1_time_pre_div", "enet1_time_cg", base + 0xa780, 16, 3); + hws[IMX7D_ENET2_REF_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet2_ref_pre_div", "enet2_ref_cg", base + 0xa800, 16, 3); + hws[IMX7D_ENET2_TIME_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet2_time_pre_div", "enet2_time_cg", base + 0xa880, 16, 3); + hws[IMX7D_ENET_PHY_REF_ROOT_PRE_DIV] = imx_clk_hw_divider2("enet_phy_ref_pre_div", "enet_phy_ref_cg", base + 0xa900, 16, 3); + hws[IMX7D_EIM_ROOT_PRE_DIV] = imx_clk_hw_divider2("eim_pre_div", "eim_cg", base + 0xa980, 16, 3); + hws[IMX7D_NAND_ROOT_PRE_DIV] = imx_clk_hw_divider2("nand_pre_div", "nand_cg", base + 0xaa00, 16, 3); + hws[IMX7D_QSPI_ROOT_PRE_DIV] = imx_clk_hw_divider2("qspi_pre_div", "qspi_cg", base + 0xaa80, 16, 3); + hws[IMX7D_USDHC1_ROOT_PRE_DIV] = imx_clk_hw_divider2("usdhc1_pre_div", "usdhc1_cg", base + 0xab00, 16, 3); + hws[IMX7D_USDHC2_ROOT_PRE_DIV] = imx_clk_hw_divider2("usdhc2_pre_div", "usdhc2_cg", base + 0xab80, 16, 3); + hws[IMX7D_USDHC3_ROOT_PRE_DIV] = imx_clk_hw_divider2("usdhc3_pre_div", "usdhc3_cg", base + 0xac00, 16, 3); + hws[IMX7D_CAN1_ROOT_PRE_DIV] = imx_clk_hw_divider2("can1_pre_div", "can1_cg", base + 0xac80, 16, 3); + hws[IMX7D_CAN2_ROOT_PRE_DIV] = imx_clk_hw_divider2("can2_pre_div", "can2_cg", base + 0xad00, 16, 3); + hws[IMX7D_I2C1_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c1_pre_div", "i2c1_cg", base + 0xad80, 16, 3); + hws[IMX7D_I2C2_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c2_pre_div", "i2c2_cg", base + 0xae00, 16, 3); + hws[IMX7D_I2C3_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c3_pre_div", "i2c3_cg", base + 0xae80, 16, 3); + hws[IMX7D_I2C4_ROOT_PRE_DIV] = imx_clk_hw_divider2("i2c4_pre_div", "i2c4_cg", base + 0xaf00, 16, 3); + hws[IMX7D_UART1_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart1_pre_div", "uart1_cg", base + 0xaf80, 16, 3); + hws[IMX7D_UART2_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart2_pre_div", "uart2_cg", base + 0xb000, 16, 3); + hws[IMX7D_UART3_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart3_pre_div", "uart3_cg", base + 0xb080, 16, 3); + hws[IMX7D_UART4_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart4_pre_div", "uart4_cg", base + 0xb100, 16, 3); + hws[IMX7D_UART5_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart5_pre_div", "uart5_cg", base + 0xb180, 16, 3); + hws[IMX7D_UART6_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart6_pre_div", "uart6_cg", base + 0xb200, 16, 3); + hws[IMX7D_UART7_ROOT_PRE_DIV] = imx_clk_hw_divider2("uart7_pre_div", "uart7_cg", base + 0xb280, 16, 3); + hws[IMX7D_ECSPI1_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi1_pre_div", "ecspi1_cg", base + 0xb300, 16, 3); + hws[IMX7D_ECSPI2_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi2_pre_div", "ecspi2_cg", base + 0xb380, 16, 3); + hws[IMX7D_ECSPI3_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi3_pre_div", "ecspi3_cg", base + 0xb400, 16, 3); + hws[IMX7D_ECSPI4_ROOT_PRE_DIV] = imx_clk_hw_divider2("ecspi4_pre_div", "ecspi4_cg", base + 0xb480, 16, 3); + hws[IMX7D_PWM1_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm1_pre_div", "pwm1_cg", base + 0xb500, 16, 3); + hws[IMX7D_PWM2_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm2_pre_div", "pwm2_cg", base + 0xb580, 16, 3); + hws[IMX7D_PWM3_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm3_pre_div", "pwm3_cg", base + 0xb600, 16, 3); + hws[IMX7D_PWM4_ROOT_PRE_DIV] = imx_clk_hw_divider2("pwm4_pre_div", "pwm4_cg", base + 0xb680, 16, 3); + hws[IMX7D_FLEXTIMER1_ROOT_PRE_DIV] = imx_clk_hw_divider2("flextimer1_pre_div", "flextimer1_cg", base + 0xb700, 16, 3); + hws[IMX7D_FLEXTIMER2_ROOT_PRE_DIV] = imx_clk_hw_divider2("flextimer2_pre_div", "flextimer2_cg", base + 0xb780, 16, 3); + hws[IMX7D_SIM1_ROOT_PRE_DIV] = imx_clk_hw_divider2("sim1_pre_div", "sim1_cg", base + 0xb800, 16, 3); + hws[IMX7D_SIM2_ROOT_PRE_DIV] = imx_clk_hw_divider2("sim2_pre_div", "sim2_cg", base + 0xb880, 16, 3); + hws[IMX7D_GPT1_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt1_pre_div", "gpt1_cg", base + 0xb900, 16, 3); + hws[IMX7D_GPT2_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt2_pre_div", "gpt2_cg", base + 0xb980, 16, 3); + hws[IMX7D_GPT3_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt3_pre_div", "gpt3_cg", base + 0xba00, 16, 3); + hws[IMX7D_GPT4_ROOT_PRE_DIV] = imx_clk_hw_divider2("gpt4_pre_div", "gpt4_cg", base + 0xba80, 16, 3); + hws[IMX7D_TRACE_ROOT_PRE_DIV] = imx_clk_hw_divider2("trace_pre_div", "trace_cg", base + 0xbb00, 16, 3); + hws[IMX7D_WDOG_ROOT_PRE_DIV] = imx_clk_hw_divider2("wdog_pre_div", "wdog_cg", base + 0xbb80, 16, 3); + hws[IMX7D_CSI_MCLK_ROOT_PRE_DIV] = imx_clk_hw_divider2("csi_mclk_pre_div", "csi_mclk_cg", base + 0xbc00, 16, 3); + hws[IMX7D_AUDIO_MCLK_ROOT_PRE_DIV] = imx_clk_hw_divider2("audio_mclk_pre_div", "audio_mclk_cg", base + 0xbc80, 16, 3); + hws[IMX7D_WRCLK_ROOT_PRE_DIV] = imx_clk_hw_divider2("wrclk_pre_div", "wrclk_cg", base + 0xbd00, 16, 3); + hws[IMX7D_CLKO1_ROOT_PRE_DIV] = imx_clk_hw_divider2("clko1_pre_div", "clko1_cg", base + 0xbd80, 16, 3); + hws[IMX7D_CLKO2_ROOT_PRE_DIV] = imx_clk_hw_divider2("clko2_pre_div", "clko2_cg", base + 0xbe00, 16, 3); + + hws[IMX7D_ARM_A7_ROOT_DIV] = imx_clk_hw_divider2("arm_a7_div", "arm_a7_cg", base + 0x8000, 0, 3); + hws[IMX7D_ARM_M4_ROOT_DIV] = imx_clk_hw_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); + hws[IMX7D_MAIN_AXI_ROOT_DIV] = imx_clk_hw_divider2("axi_post_div", "axi_pre_div", base + 0x8800, 0, 6); + hws[IMX7D_DISP_AXI_ROOT_DIV] = imx_clk_hw_divider2("disp_axi_post_div", "disp_axi_pre_div", base + 0x8880, 0, 6); + hws[IMX7D_ENET_AXI_ROOT_DIV] = imx_clk_hw_divider2("enet_axi_post_div", "enet_axi_pre_div", base + 0x8900, 0, 6); + hws[IMX7D_NAND_USDHC_BUS_ROOT_CLK] = imx_clk_hw_divider2("nand_usdhc_root_clk", "nand_usdhc_pre_div", base + 0x8980, 0, 6); + hws[IMX7D_AHB_CHANNEL_ROOT_DIV] = imx_clk_hw_divider2("ahb_root_clk", "ahb_pre_div", base + 0x9000, 0, 6); + hws[IMX7D_IPG_ROOT_CLK] = imx_clk_hw_divider_flags("ipg_root_clk", "ahb_root_clk", base + 0x9080, 0, 2, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE | CLK_SET_RATE_PARENT); + hws[IMX7D_DRAM_ROOT_DIV] = imx_clk_hw_divider2("dram_post_div", "dram_cg", base + 0x9880, 0, 3); + hws[IMX7D_DRAM_PHYM_ALT_ROOT_DIV] = imx_clk_hw_divider2("dram_phym_alt_post_div", "dram_phym_alt_pre_div", base + 0xa000, 0, 3); + hws[IMX7D_DRAM_ALT_ROOT_DIV] = imx_clk_hw_divider2("dram_alt_post_div", "dram_alt_pre_div", base + 0xa080, 0, 3); + hws[IMX7D_USB_HSIC_ROOT_DIV] = imx_clk_hw_divider2("usb_hsic_post_div", "usb_hsic_pre_div", base + 0xa100, 0, 6); + hws[IMX7D_PCIE_CTRL_ROOT_DIV] = imx_clk_hw_divider2("pcie_ctrl_post_div", "pcie_ctrl_pre_div", base + 0xa180, 0, 6); + hws[IMX7D_PCIE_PHY_ROOT_DIV] = imx_clk_hw_divider2("pcie_phy_post_div", "pcie_phy_pre_div", base + 0xa200, 0, 6); + hws[IMX7D_EPDC_PIXEL_ROOT_DIV] = imx_clk_hw_divider2("epdc_pixel_post_div", "epdc_pixel_pre_div", base + 0xa280, 0, 6); + hws[IMX7D_LCDIF_PIXEL_ROOT_DIV] = imx_clk_hw_divider2("lcdif_pixel_post_div", "lcdif_pixel_pre_div", base + 0xa300, 0, 6); + hws[IMX7D_MIPI_DSI_ROOT_DIV] = imx_clk_hw_divider2("mipi_dsi_post_div", "mipi_dsi_pre_div", base + 0xa380, 0, 6); + hws[IMX7D_MIPI_CSI_ROOT_DIV] = imx_clk_hw_divider2("mipi_csi_post_div", "mipi_csi_pre_div", base + 0xa400, 0, 6); + hws[IMX7D_MIPI_DPHY_ROOT_DIV] = imx_clk_hw_divider2("mipi_dphy_post_div", "mipi_dphy_pre_div", base + 0xa480, 0, 6); + hws[IMX7D_SAI1_ROOT_DIV] = imx_clk_hw_divider2("sai1_post_div", "sai1_pre_div", base + 0xa500, 0, 6); + hws[IMX7D_SAI2_ROOT_DIV] = imx_clk_hw_divider2("sai2_post_div", "sai2_pre_div", base + 0xa580, 0, 6); + hws[IMX7D_SAI3_ROOT_DIV] = imx_clk_hw_divider2("sai3_post_div", "sai3_pre_div", base + 0xa600, 0, 6); + hws[IMX7D_SPDIF_ROOT_DIV] = imx_clk_hw_divider2("spdif_post_div", "spdif_pre_div", base + 0xa680, 0, 6); + hws[IMX7D_ENET1_REF_ROOT_DIV] = imx_clk_hw_divider2("enet1_ref_post_div", "enet1_ref_pre_div", base + 0xa700, 0, 6); + hws[IMX7D_ENET1_TIME_ROOT_DIV] = imx_clk_hw_divider2("enet1_time_post_div", "enet1_time_pre_div", base + 0xa780, 0, 6); + hws[IMX7D_ENET2_REF_ROOT_DIV] = imx_clk_hw_divider2("enet2_ref_post_div", "enet2_ref_pre_div", base + 0xa800, 0, 6); + hws[IMX7D_ENET2_TIME_ROOT_DIV] = imx_clk_hw_divider2("enet2_time_post_div", "enet2_time_pre_div", base + 0xa880, 0, 6); + hws[IMX7D_ENET_PHY_REF_ROOT_CLK] = imx_clk_hw_divider2("enet_phy_ref_root_clk", "enet_phy_ref_pre_div", base + 0xa900, 0, 6); + hws[IMX7D_EIM_ROOT_DIV] = imx_clk_hw_divider2("eim_post_div", "eim_pre_div", base + 0xa980, 0, 6); + hws[IMX7D_NAND_ROOT_CLK] = imx_clk_hw_divider2("nand_root_clk", "nand_pre_div", base + 0xaa00, 0, 6); + hws[IMX7D_QSPI_ROOT_DIV] = imx_clk_hw_divider2("qspi_post_div", "qspi_pre_div", base + 0xaa80, 0, 6); + hws[IMX7D_USDHC1_ROOT_DIV] = imx_clk_hw_divider2("usdhc1_post_div", "usdhc1_pre_div", base + 0xab00, 0, 6); + hws[IMX7D_USDHC2_ROOT_DIV] = imx_clk_hw_divider2("usdhc2_post_div", "usdhc2_pre_div", base + 0xab80, 0, 6); + hws[IMX7D_USDHC3_ROOT_DIV] = imx_clk_hw_divider2("usdhc3_post_div", "usdhc3_pre_div", base + 0xac00, 0, 6); + hws[IMX7D_CAN1_ROOT_DIV] = imx_clk_hw_divider2("can1_post_div", "can1_pre_div", base + 0xac80, 0, 6); + hws[IMX7D_CAN2_ROOT_DIV] = imx_clk_hw_divider2("can2_post_div", "can2_pre_div", base + 0xad00, 0, 6); + hws[IMX7D_I2C1_ROOT_DIV] = imx_clk_hw_divider2("i2c1_post_div", "i2c1_pre_div", base + 0xad80, 0, 6); + hws[IMX7D_I2C2_ROOT_DIV] = imx_clk_hw_divider2("i2c2_post_div", "i2c2_pre_div", base + 0xae00, 0, 6); + hws[IMX7D_I2C3_ROOT_DIV] = imx_clk_hw_divider2("i2c3_post_div", "i2c3_pre_div", base + 0xae80, 0, 6); + hws[IMX7D_I2C4_ROOT_DIV] = imx_clk_hw_divider2("i2c4_post_div", "i2c4_pre_div", base + 0xaf00, 0, 6); + hws[IMX7D_UART1_ROOT_DIV] = imx_clk_hw_divider2("uart1_post_div", "uart1_pre_div", base + 0xaf80, 0, 6); + hws[IMX7D_UART2_ROOT_DIV] = imx_clk_hw_divider2("uart2_post_div", "uart2_pre_div", base + 0xb000, 0, 6); + hws[IMX7D_UART3_ROOT_DIV] = imx_clk_hw_divider2("uart3_post_div", "uart3_pre_div", base + 0xb080, 0, 6); + hws[IMX7D_UART4_ROOT_DIV] = imx_clk_hw_divider2("uart4_post_div", "uart4_pre_div", base + 0xb100, 0, 6); + hws[IMX7D_UART5_ROOT_DIV] = imx_clk_hw_divider2("uart5_post_div", "uart5_pre_div", base + 0xb180, 0, 6); + hws[IMX7D_UART6_ROOT_DIV] = imx_clk_hw_divider2("uart6_post_div", "uart6_pre_div", base + 0xb200, 0, 6); + hws[IMX7D_UART7_ROOT_DIV] = imx_clk_hw_divider2("uart7_post_div", "uart7_pre_div", base + 0xb280, 0, 6); + hws[IMX7D_ECSPI1_ROOT_DIV] = imx_clk_hw_divider2("ecspi1_post_div", "ecspi1_pre_div", base + 0xb300, 0, 6); + hws[IMX7D_ECSPI2_ROOT_DIV] = imx_clk_hw_divider2("ecspi2_post_div", "ecspi2_pre_div", base + 0xb380, 0, 6); + hws[IMX7D_ECSPI3_ROOT_DIV] = imx_clk_hw_divider2("ecspi3_post_div", "ecspi3_pre_div", base + 0xb400, 0, 6); + hws[IMX7D_ECSPI4_ROOT_DIV] = imx_clk_hw_divider2("ecspi4_post_div", "ecspi4_pre_div", base + 0xb480, 0, 6); + hws[IMX7D_PWM1_ROOT_DIV] = imx_clk_hw_divider2("pwm1_post_div", "pwm1_pre_div", base + 0xb500, 0, 6); + hws[IMX7D_PWM2_ROOT_DIV] = imx_clk_hw_divider2("pwm2_post_div", "pwm2_pre_div", base + 0xb580, 0, 6); + hws[IMX7D_PWM3_ROOT_DIV] = imx_clk_hw_divider2("pwm3_post_div", "pwm3_pre_div", base + 0xb600, 0, 6); + hws[IMX7D_PWM4_ROOT_DIV] = imx_clk_hw_divider2("pwm4_post_div", "pwm4_pre_div", base + 0xb680, 0, 6); + hws[IMX7D_FLEXTIMER1_ROOT_DIV] = imx_clk_hw_divider2("flextimer1_post_div", "flextimer1_pre_div", base + 0xb700, 0, 6); + hws[IMX7D_FLEXTIMER2_ROOT_DIV] = imx_clk_hw_divider2("flextimer2_post_div", "flextimer2_pre_div", base + 0xb780, 0, 6); + hws[IMX7D_SIM1_ROOT_DIV] = imx_clk_hw_divider2("sim1_post_div", "sim1_pre_div", base + 0xb800, 0, 6); + hws[IMX7D_SIM2_ROOT_DIV] = imx_clk_hw_divider2("sim2_post_div", "sim2_pre_div", base + 0xb880, 0, 6); + hws[IMX7D_GPT1_ROOT_DIV] = imx_clk_hw_divider2("gpt1_post_div", "gpt1_pre_div", base + 0xb900, 0, 6); + hws[IMX7D_GPT2_ROOT_DIV] = imx_clk_hw_divider2("gpt2_post_div", "gpt2_pre_div", base + 0xb980, 0, 6); + hws[IMX7D_GPT3_ROOT_DIV] = imx_clk_hw_divider2("gpt3_post_div", "gpt3_pre_div", base + 0xba00, 0, 6); + hws[IMX7D_GPT4_ROOT_DIV] = imx_clk_hw_divider2("gpt4_post_div", "gpt4_pre_div", base + 0xba80, 0, 6); + hws[IMX7D_TRACE_ROOT_DIV] = imx_clk_hw_divider2("trace_post_div", "trace_pre_div", base + 0xbb00, 0, 6); + hws[IMX7D_WDOG_ROOT_DIV] = imx_clk_hw_divider2("wdog_post_div", "wdog_pre_div", base + 0xbb80, 0, 6); + hws[IMX7D_CSI_MCLK_ROOT_DIV] = imx_clk_hw_divider2("csi_mclk_post_div", "csi_mclk_pre_div", base + 0xbc00, 0, 6); + hws[IMX7D_AUDIO_MCLK_ROOT_DIV] = imx_clk_hw_divider2("audio_mclk_post_div", "audio_mclk_pre_div", base + 0xbc80, 0, 6); + hws[IMX7D_WRCLK_ROOT_DIV] = imx_clk_hw_divider2("wrclk_post_div", "wrclk_pre_div", base + 0xbd00, 0, 6); + hws[IMX7D_CLKO1_ROOT_DIV] = imx_clk_hw_divider2("clko1_post_div", "clko1_pre_div", base + 0xbd80, 0, 6); + hws[IMX7D_CLKO2_ROOT_DIV] = imx_clk_hw_divider2("clko2_post_div", "clko2_pre_div", base + 0xbe00, 0, 6); + + hws[IMX7D_ARM_A7_ROOT_CLK] = imx_clk_hw_gate2_flags("arm_a7_root_clk", "arm_a7_div", base + 0x4000, 0, CLK_OPS_PARENT_ENABLE); + hws[IMX7D_ARM_M4_ROOT_CLK] = imx_clk_hw_gate4("arm_m4_root_clk", "arm_m4_div", base + 0x4010, 0); + hws[IMX7D_MAIN_AXI_ROOT_CLK] = imx_clk_hw_gate2_flags("main_axi_root_clk", "axi_post_div", base + 0x4040, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); + hws[IMX7D_DISP_AXI_ROOT_CLK] = imx_clk_hw_gate4("disp_axi_root_clk", "disp_axi_post_div", base + 0x4050, 0); + hws[IMX7D_ENET_AXI_ROOT_CLK] = imx_clk_hw_gate4("enet_axi_root_clk", "enet_axi_post_div", base + 0x4060, 0); + hws[IMX7D_OCRAM_CLK] = imx_clk_hw_gate4("ocram_clk", "main_axi_root_clk", base + 0x4110, 0); + hws[IMX7D_OCRAM_S_CLK] = imx_clk_hw_gate4("ocram_s_clk", "ahb_root_clk", base + 0x4120, 0); + hws[IMX7D_DRAM_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_root_clk", "dram_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); + hws[IMX7D_DRAM_PHYM_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_phym_root_clk", "dram_phym_cg", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); + hws[IMX7D_DRAM_PHYM_ALT_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_phym_alt_root_clk", "dram_phym_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); + hws[IMX7D_DRAM_ALT_ROOT_CLK] = imx_clk_hw_gate2_flags("dram_alt_root_clk", "dram_alt_post_div", base + 0x4130, 0, CLK_IS_CRITICAL | CLK_OPS_PARENT_ENABLE); + hws[IMX7D_OCOTP_CLK] = imx_clk_hw_gate4("ocotp_clk", "ipg_root_clk", base + 0x4230, 0); + hws[IMX7D_SNVS_CLK] = imx_clk_hw_gate4("snvs_clk", "ipg_root_clk", base + 0x4250, 0); + hws[IMX7D_MU_ROOT_CLK] = imx_clk_hw_gate4("mu_root_clk", "ipg_root_clk", base + 0x4270, 0); + hws[IMX7D_CAAM_CLK] = imx_clk_hw_gate4("caam_clk", "ipg_root_clk", base + 0x4240, 0); + hws[IMX7D_USB_HSIC_ROOT_CLK] = imx_clk_hw_gate4("usb_hsic_root_clk", "usb_hsic_post_div", base + 0x4690, 0); + hws[IMX7D_SDMA_CORE_CLK] = imx_clk_hw_gate4("sdma_root_clk", "ahb_root_clk", base + 0x4480, 0); + hws[IMX7D_PCIE_CTRL_ROOT_CLK] = imx_clk_hw_gate4("pcie_ctrl_root_clk", "pcie_ctrl_post_div", base + 0x4600, 0); + hws[IMX7D_PCIE_PHY_ROOT_CLK] = imx_clk_hw_gate4("pcie_phy_root_clk", "pcie_phy_post_div", base + 0x4600, 0); + hws[IMX7D_EPDC_PIXEL_ROOT_CLK] = imx_clk_hw_gate4("epdc_pixel_root_clk", "epdc_pixel_post_div", base + 0x44a0, 0); + hws[IMX7D_LCDIF_PIXEL_ROOT_CLK] = imx_clk_hw_gate4("lcdif_pixel_root_clk", "lcdif_pixel_post_div", base + 0x44b0, 0); + hws[IMX7D_MIPI_DSI_ROOT_CLK] = imx_clk_hw_gate4("mipi_dsi_root_clk", "mipi_dsi_post_div", base + 0x4650, 0); + hws[IMX7D_MIPI_CSI_ROOT_CLK] = imx_clk_hw_gate4("mipi_csi_root_clk", "mipi_csi_post_div", base + 0x4640, 0); + hws[IMX7D_MIPI_DPHY_ROOT_CLK] = imx_clk_hw_gate4("mipi_dphy_root_clk", "mipi_dphy_post_div", base + 0x4660, 0); + hws[IMX7D_ENET1_IPG_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet1_ipg_root_clk", "enet_axi_post_div", base + 0x4700, 0, &share_count_enet1); + hws[IMX7D_ENET1_TIME_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet1_time_root_clk", "enet1_time_post_div", base + 0x4700, 0, &share_count_enet1); + hws[IMX7D_ENET2_IPG_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet2_ipg_root_clk", "enet_axi_post_div", base + 0x4710, 0, &share_count_enet2); + hws[IMX7D_ENET2_TIME_ROOT_CLK] = imx_clk_hw_gate2_shared2("enet2_time_root_clk", "enet2_time_post_div", base + 0x4710, 0, &share_count_enet2); + hws[IMX7D_SAI1_ROOT_CLK] = imx_clk_hw_gate2_shared2("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0, &share_count_sai1); + hws[IMX7D_SAI1_IPG_CLK] = imx_clk_hw_gate2_shared2("sai1_ipg_clk", "ipg_root_clk", base + 0x48c0, 0, &share_count_sai1); + hws[IMX7D_SAI2_ROOT_CLK] = imx_clk_hw_gate2_shared2("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0, &share_count_sai2); + hws[IMX7D_SAI2_IPG_CLK] = imx_clk_hw_gate2_shared2("sai2_ipg_clk", "ipg_root_clk", base + 0x48d0, 0, &share_count_sai2); + hws[IMX7D_SAI3_ROOT_CLK] = imx_clk_hw_gate2_shared2("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0, &share_count_sai3); + hws[IMX7D_SAI3_IPG_CLK] = imx_clk_hw_gate2_shared2("sai3_ipg_clk", "ipg_root_clk", base + 0x48e0, 0, &share_count_sai3); + hws[IMX7D_SPDIF_ROOT_CLK] = imx_clk_hw_gate4("spdif_root_clk", "spdif_post_div", base + 0x44d0, 0); + hws[IMX7D_EIM_ROOT_CLK] = imx_clk_hw_gate4("eim_root_clk", "eim_post_div", base + 0x4160, 0); + hws[IMX7D_NAND_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_rawnand_clk", "nand_root_clk", base + 0x4140, 0, &share_count_nand); + hws[IMX7D_NAND_USDHC_BUS_RAWNAND_CLK] = imx_clk_hw_gate2_shared2("nand_usdhc_rawnand_clk", "nand_usdhc_root_clk", base + 0x4140, 0, &share_count_nand); + hws[IMX7D_QSPI_ROOT_CLK] = imx_clk_hw_gate4("qspi_root_clk", "qspi_post_div", base + 0x4150, 0); + hws[IMX7D_USDHC1_ROOT_CLK] = imx_clk_hw_gate4("usdhc1_root_clk", "usdhc1_post_div", base + 0x46c0, 0); + hws[IMX7D_USDHC2_ROOT_CLK] = imx_clk_hw_gate4("usdhc2_root_clk", "usdhc2_post_div", base + 0x46d0, 0); + hws[IMX7D_USDHC3_ROOT_CLK] = imx_clk_hw_gate4("usdhc3_root_clk", "usdhc3_post_div", base + 0x46e0, 0); + hws[IMX7D_CAN1_ROOT_CLK] = imx_clk_hw_gate4("can1_root_clk", "can1_post_div", base + 0x4740, 0); + hws[IMX7D_CAN2_ROOT_CLK] = imx_clk_hw_gate4("can2_root_clk", "can2_post_div", base + 0x4750, 0); + hws[IMX7D_I2C1_ROOT_CLK] = imx_clk_hw_gate4("i2c1_root_clk", "i2c1_post_div", base + 0x4880, 0); + hws[IMX7D_I2C2_ROOT_CLK] = imx_clk_hw_gate4("i2c2_root_clk", "i2c2_post_div", base + 0x4890, 0); + hws[IMX7D_I2C3_ROOT_CLK] = imx_clk_hw_gate4("i2c3_root_clk", "i2c3_post_div", base + 0x48a0, 0); + hws[IMX7D_I2C4_ROOT_CLK] = imx_clk_hw_gate4("i2c4_root_clk", "i2c4_post_div", base + 0x48b0, 0); + hws[IMX7D_UART1_ROOT_CLK] = imx_clk_hw_gate4("uart1_root_clk", "uart1_post_div", base + 0x4940, 0); + hws[IMX7D_UART2_ROOT_CLK] = imx_clk_hw_gate4("uart2_root_clk", "uart2_post_div", base + 0x4950, 0); + hws[IMX7D_UART3_ROOT_CLK] = imx_clk_hw_gate4("uart3_root_clk", "uart3_post_div", base + 0x4960, 0); + hws[IMX7D_UART4_ROOT_CLK] = imx_clk_hw_gate4("uart4_root_clk", "uart4_post_div", base + 0x4970, 0); + hws[IMX7D_UART5_ROOT_CLK] = imx_clk_hw_gate4("uart5_root_clk", "uart5_post_div", base + 0x4980, 0); + hws[IMX7D_UART6_ROOT_CLK] = imx_clk_hw_gate4("uart6_root_clk", "uart6_post_div", base + 0x4990, 0); + hws[IMX7D_UART7_ROOT_CLK] = imx_clk_hw_gate4("uart7_root_clk", "uart7_post_div", base + 0x49a0, 0); + hws[IMX7D_ECSPI1_ROOT_CLK] = imx_clk_hw_gate4("ecspi1_root_clk", "ecspi1_post_div", base + 0x4780, 0); + hws[IMX7D_ECSPI2_ROOT_CLK] = imx_clk_hw_gate4("ecspi2_root_clk", "ecspi2_post_div", base + 0x4790, 0); + hws[IMX7D_ECSPI3_ROOT_CLK] = imx_clk_hw_gate4("ecspi3_root_clk", "ecspi3_post_div", base + 0x47a0, 0); + hws[IMX7D_ECSPI4_ROOT_CLK] = imx_clk_hw_gate4("ecspi4_root_clk", "ecspi4_post_div", base + 0x47b0, 0); + hws[IMX7D_PWM1_ROOT_CLK] = imx_clk_hw_gate4("pwm1_root_clk", "pwm1_post_div", base + 0x4840, 0); + hws[IMX7D_PWM2_ROOT_CLK] = imx_clk_hw_gate4("pwm2_root_clk", "pwm2_post_div", base + 0x4850, 0); + hws[IMX7D_PWM3_ROOT_CLK] = imx_clk_hw_gate4("pwm3_root_clk", "pwm3_post_div", base + 0x4860, 0); + hws[IMX7D_PWM4_ROOT_CLK] = imx_clk_hw_gate4("pwm4_root_clk", "pwm4_post_div", base + 0x4870, 0); + hws[IMX7D_FLEXTIMER1_ROOT_CLK] = imx_clk_hw_gate4("flextimer1_root_clk", "flextimer1_post_div", base + 0x4800, 0); + hws[IMX7D_FLEXTIMER2_ROOT_CLK] = imx_clk_hw_gate4("flextimer2_root_clk", "flextimer2_post_div", base + 0x4810, 0); + hws[IMX7D_SIM1_ROOT_CLK] = imx_clk_hw_gate4("sim1_root_clk", "sim1_post_div", base + 0x4900, 0); + hws[IMX7D_SIM2_ROOT_CLK] = imx_clk_hw_gate4("sim2_root_clk", "sim2_post_div", base + 0x4910, 0); + hws[IMX7D_GPT1_ROOT_CLK] = imx_clk_hw_gate4("gpt1_root_clk", "gpt1_post_div", base + 0x47c0, 0); + hws[IMX7D_GPT2_ROOT_CLK] = imx_clk_hw_gate4("gpt2_root_clk", "gpt2_post_div", base + 0x47d0, 0); + hws[IMX7D_GPT3_ROOT_CLK] = imx_clk_hw_gate4("gpt3_root_clk", "gpt3_post_div", base + 0x47e0, 0); + hws[IMX7D_GPT4_ROOT_CLK] = imx_clk_hw_gate4("gpt4_root_clk", "gpt4_post_div", base + 0x47f0, 0); + hws[IMX7D_TRACE_ROOT_CLK] = imx_clk_hw_gate4("trace_root_clk", "trace_post_div", base + 0x4300, 0); + hws[IMX7D_WDOG1_ROOT_CLK] = imx_clk_hw_gate4("wdog1_root_clk", "wdog_post_div", base + 0x49c0, 0); + hws[IMX7D_WDOG2_ROOT_CLK] = imx_clk_hw_gate4("wdog2_root_clk", "wdog_post_div", base + 0x49d0, 0); + hws[IMX7D_WDOG3_ROOT_CLK] = imx_clk_hw_gate4("wdog3_root_clk", "wdog_post_div", base + 0x49e0, 0); + hws[IMX7D_WDOG4_ROOT_CLK] = imx_clk_hw_gate4("wdog4_root_clk", "wdog_post_div", base + 0x49f0, 0); + hws[IMX7D_KPP_ROOT_CLK] = imx_clk_hw_gate4("kpp_root_clk", "ipg_root_clk", base + 0x4aa0, 0); + hws[IMX7D_CSI_MCLK_ROOT_CLK] = imx_clk_hw_gate4("csi_mclk_root_clk", "csi_mclk_post_div", base + 0x4490, 0); + hws[IMX7D_AUDIO_MCLK_ROOT_CLK] = imx_clk_hw_gate4("audio_mclk_root_clk", "audio_mclk_post_div", base + 0x4790, 0); + hws[IMX7D_WRCLK_ROOT_CLK] = imx_clk_hw_gate4("wrclk_root_clk", "wrclk_post_div", base + 0x47a0, 0); + hws[IMX7D_USB_CTRL_CLK] = imx_clk_hw_gate4("usb_ctrl_clk", "ahb_root_clk", base + 0x4680, 0); + hws[IMX7D_USB_PHY1_CLK] = imx_clk_hw_gate4("usb_phy1_clk", "pll_usb1_main_clk", base + 0x46a0, 0); + hws[IMX7D_USB_PHY2_CLK] = imx_clk_hw_gate4("usb_phy2_clk", "pll_usb_main_clk", base + 0x46b0, 0); + hws[IMX7D_ADC_ROOT_CLK] = imx_clk_hw_gate4("adc_root_clk", "ipg_root_clk", base + 0x4200, 0); + + hws[IMX7D_GPT_3M_CLK] = imx_clk_hw_fixed_factor("gpt_3m", "osc", 1, 8); + + hws[IMX7D_CLK_ARM] = imx_clk_hw_cpu("arm", "arm_a7_root_clk", + hws[IMX7D_ARM_A7_ROOT_CLK]->clk, + hws[IMX7D_ARM_A7_ROOT_SRC]->clk, + hws[IMX7D_PLL_ARM_MAIN_CLK]->clk, + hws[IMX7D_PLL_SYS_MAIN_CLK]->clk); + + imx_check_clk_hws(hws, IMX7D_CLK_END); + + of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data); + + clk_set_parent(hws[IMX7D_PLL_ARM_MAIN_BYPASS]->clk, hws[IMX7D_PLL_ARM_MAIN]->clk); + clk_set_parent(hws[IMX7D_PLL_DRAM_MAIN_BYPASS]->clk, hws[IMX7D_PLL_DRAM_MAIN]->clk); + clk_set_parent(hws[IMX7D_PLL_SYS_MAIN_BYPASS]->clk, hws[IMX7D_PLL_SYS_MAIN]->clk); + clk_set_parent(hws[IMX7D_PLL_ENET_MAIN_BYPASS]->clk, hws[IMX7D_PLL_ENET_MAIN]->clk); + clk_set_parent(hws[IMX7D_PLL_AUDIO_MAIN_BYPASS]->clk, hws[IMX7D_PLL_AUDIO_MAIN]->clk); + clk_set_parent(hws[IMX7D_PLL_VIDEO_MAIN_BYPASS]->clk, hws[IMX7D_PLL_VIDEO_MAIN]->clk); + + clk_set_parent(hws[IMX7D_MIPI_CSI_ROOT_SRC]->clk, hws[IMX7D_PLL_SYS_PFD3_CLK]->clk); /* use old gpt clk setting, gpt1 root clk must be twice as gpt counter freq */ - clk_set_parent(clks[IMX7D_GPT1_ROOT_SRC], clks[IMX7D_OSC_24M_CLK]); + clk_set_parent(hws[IMX7D_GPT1_ROOT_SRC]->clk, hws[IMX7D_OSC_24M_CLK]->clk); /* Set clock rate for USBPHY, the USB_PLL at CCM is from USBOTG2 */ - clks[IMX7D_USB1_MAIN_480M_CLK] = imx_clk_fixed_factor("pll_usb1_main_clk", "osc", 20, 1); - clks[IMX7D_USB_MAIN_480M_CLK] = imx_clk_fixed_factor("pll_usb_main_clk", "osc", 20, 1); + hws[IMX7D_USB1_MAIN_480M_CLK] = imx_clk_hw_fixed_factor("pll_usb1_main_clk", "osc", 20, 1); + hws[IMX7D_USB_MAIN_480M_CLK] = imx_clk_hw_fixed_factor("pll_usb_main_clk", "osc", 20, 1); + + for (i = 0; i < ARRAY_SIZE(uart_clk_ids); i++) { + int index = uart_clk_ids[i]; + + uart_clks[i] = &hws[index]->clk; + } + imx_register_uart_clocks(uart_clks); diff --git a/drivers/clk/imx/clk-imx7ulp.c b/drivers/clk/imx/clk-imx7ulp.c index 66682100f14c..42e4667f22fd 100644 --- a/drivers/clk/imx/clk-imx7ulp.c +++ b/drivers/clk/imx/clk-imx7ulp.c @@ -115,7 +115,7 @@ static void __init imx7ulp_clk_scg1_init(struct device_node *np) clks[IMX7ULP_CLK_NIC0_DIV] = imx_clk_hw_divider_flags("nic0_clk", "nic_sel", base + 0x40, 24, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); clks[IMX7ULP_CLK_NIC1_DIV] = imx_clk_hw_divider_flags("nic1_clk", "nic0_clk", base + 0x40, 16, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); - clks[IMX7ULP_CLK_NIC1_BUS_DIV] = imx_clk_hw_divider_flags("nic1_bus_clk", "nic1_clk", base + 0x40, 4, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); + clks[IMX7ULP_CLK_NIC1_BUS_DIV] = imx_clk_hw_divider_flags("nic1_bus_clk", "nic0_clk", base + 0x40, 4, 4, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); clks[IMX7ULP_CLK_GPU_DIV] = imx_clk_hw_divider("gpu_clk", "nic0_clk", base + 0x40, 20, 4); diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c index 122a81ab8e48..6b8e75df994d 100644 --- a/drivers/clk/imx/clk-imx8mm.c +++ b/drivers/clk/imx/clk-imx8mm.c @@ -288,6 +288,9 @@ static const char *imx8mm_usb_core_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pl static const char *imx8mm_usb_phy_sels[] = {"osc_24m", "sys_pll1_100m", "sys_pll1_40m", "sys_pll2_100m", "sys_pll2_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", }; +static const char *imx8mm_gic_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll2_100m", + "sys_pll1_800m", "clk_ext2", "clk_ext4", "audio_pll2_out" }; + static const char *imx8mm_ecspi1_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_40m", "sys_pll1_160m", "sys_pll1_800m", "sys_pll3_out", "sys_pll2_250m", "audio_pll2_out", }; @@ -325,7 +328,7 @@ static const char *imx8mm_dsi_dbi_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll "sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", }; static const char *imx8mm_usdhc3_sels[] = {"osc_24m", "sys_pll1_400m", "sys_pll1_800m", "sys_pll2_500m", - "sys_pll3_out", "sys_pll1_266m", "audio_pll2_clk", "sys_pll1_100m", }; + "sys_pll3_out", "sys_pll1_266m", "audio_pll2_out", "sys_pll1_100m", }; static const char *imx8mm_csi1_core_sels[] = {"osc_24m", "sys_pll1_266m", "sys_pll2_250m", "sys_pll1_800m", "sys_pll2_1000m", "sys_pll3_out", "audio_pll2_out", "video_pll1_out", }; @@ -361,11 +364,11 @@ static const char *imx8mm_pdm_sels[] = {"osc_24m", "sys_pll2_100m", "audio_pll1_ "sys_pll2_1000m", "sys_pll3_out", "clk_ext3", "audio_pll2_out", }; static const char *imx8mm_vpu_h1_sels[] = {"osc_24m", "vpu_pll_out", "sys_pll1_800m", "sys_pll2_1000m", - "audio_pll2_clk", "sys_pll2_125m", "sys_pll3_clk", "audio_pll1_out", }; + "audio_pll2_out", "sys_pll2_125m", "sys_pll3_clk", "audio_pll1_out", }; static const char *imx8mm_dram_core_sels[] = {"dram_pll_out", "dram_alt_root", }; -static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m", "sys_pll1_200m", "audio_pll2_clk", +static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "osc_27m", "sys_pll1_200m", "audio_pll2_out", "vpu_pll", "sys_pll1_80m", }; static struct clk *clks[IMX8MM_CLK_END]; @@ -523,7 +526,7 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node) /* IP */ clks[IMX8MM_CLK_DRAM_ALT] = imx8m_clk_composite("dram_alt", imx8mm_dram_alt_sels, base + 0xa000); - clks[IMX8MM_CLK_DRAM_APB] = imx8m_clk_composite("dram_apb", imx8mm_dram_apb_sels, base + 0xa080); + clks[IMX8MM_CLK_DRAM_APB] = imx8m_clk_composite_critical("dram_apb", imx8mm_dram_apb_sels, base + 0xa080); clks[IMX8MM_CLK_VPU_G1] = imx8m_clk_composite("vpu_g1", imx8mm_vpu_g1_sels, base + 0xa100); clks[IMX8MM_CLK_VPU_G2] = imx8m_clk_composite("vpu_g2", imx8mm_vpu_g2_sels, base + 0xa180); clks[IMX8MM_CLK_DISP_DTRC] = imx8m_clk_composite("disp_dtrc", imx8mm_disp_dtrc_sels, base + 0xa200); @@ -558,6 +561,7 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node) clks[IMX8MM_CLK_UART4] = imx8m_clk_composite("uart4", imx8mm_uart4_sels, base + 0xb080); clks[IMX8MM_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mm_usb_core_sels, base + 0xb100); clks[IMX8MM_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mm_usb_phy_sels, base + 0xb180); + clks[IMX8MM_CLK_GIC] = imx8m_clk_composite_critical("gic", imx8mm_gic_sels, base + 0xb200); clks[IMX8MM_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mm_ecspi1_sels, base + 0xb280); clks[IMX8MM_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mm_ecspi2_sels, base + 0xb300); clks[IMX8MM_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mm_pwm1_sels, base + 0xb380); @@ -590,6 +594,11 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node) clks[IMX8MM_CLK_ECSPI2_ROOT] = imx_clk_gate4("ecspi2_root_clk", "ecspi2", base + 0x4080, 0); clks[IMX8MM_CLK_ECSPI3_ROOT] = imx_clk_gate4("ecspi3_root_clk", "ecspi3", base + 0x4090, 0); clks[IMX8MM_CLK_ENET1_ROOT] = imx_clk_gate4("enet1_root_clk", "enet_axi", base + 0x40a0, 0); + clks[IMX8MM_CLK_GPIO1_ROOT] = imx_clk_gate4("gpio1_root_clk", "ipg_root", base + 0x40b0, 0); + clks[IMX8MM_CLK_GPIO2_ROOT] = imx_clk_gate4("gpio2_root_clk", "ipg_root", base + 0x40c0, 0); + clks[IMX8MM_CLK_GPIO3_ROOT] = imx_clk_gate4("gpio3_root_clk", "ipg_root", base + 0x40d0, 0); + clks[IMX8MM_CLK_GPIO4_ROOT] = imx_clk_gate4("gpio4_root_clk", "ipg_root", base + 0x40e0, 0); + clks[IMX8MM_CLK_GPIO5_ROOT] = imx_clk_gate4("gpio5_root_clk", "ipg_root", base + 0x40f0, 0); clks[IMX8MM_CLK_GPT1_ROOT] = imx_clk_gate4("gpt1_root_clk", "gpt1", base + 0x4100, 0); clks[IMX8MM_CLK_I2C1_ROOT] = imx_clk_gate4("i2c1_root_clk", "i2c1", base + 0x4170, 0); clks[IMX8MM_CLK_I2C2_ROOT] = imx_clk_gate4("i2c2_root_clk", "i2c2", base + 0x4180, 0); @@ -617,6 +626,7 @@ static int __init imx8mm_clocks_init(struct device_node *ccm_node) clks[IMX8MM_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); clks[IMX8MM_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); clks[IMX8MM_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); + clks[IMX8MM_CLK_SNVS_ROOT] = imx_clk_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0); clks[IMX8MM_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); clks[IMX8MM_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); clks[IMX8MM_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index daf1841b2adb..d407a07e7e6d 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -192,6 +192,9 @@ static const char * const imx8mq_usb_core_sels[] = {"osc_25m", "sys1_pll_100m", static const char * const imx8mq_usb_phy_sels[] = {"osc_25m", "sys1_pll_100m", "sys1_pll_40m", "sys2_pll_100m", "sys2_pll_200m", "clk_ext2", "clk_ext3", "audio_pll2_out", }; +static const char * const imx8mq_gic_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys2_pll_100m", + "sys2_pll_200m", "clk_ext2", "clk_ext3", "audio_pll2_out" }; + static const char * const imx8mq_ecspi1_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_40m", "sys1_pll_160m", "sys1_pll_800m", "sys3_pll2_out", "sys2_pll_250m", "audio_pll2_out", }; @@ -269,13 +272,20 @@ static const char * const imx8mq_clko2_sels[] = {"osc_25m", "sys2_pll_200m", "sy static struct clk_onecell_data clk_data; +static struct clk ** const uart_clks[] = { + &clks[IMX8MQ_CLK_UART1_ROOT], + &clks[IMX8MQ_CLK_UART2_ROOT], + &clks[IMX8MQ_CLK_UART3_ROOT], + &clks[IMX8MQ_CLK_UART4_ROOT], + NULL +}; + static int imx8mq_clocks_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; void __iomem *base; int err; - int i; clks[IMX8MQ_CLK_DUMMY] = imx_clk_fixed("dummy", 0); clks[IMX8MQ_CLK_32K] = of_clk_get_by_name(np, "ckil"); @@ -358,9 +368,9 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_SYS2_PLL_1000M] = imx_clk_fixed_factor("sys2_pll_1000m", "sys2_pll_out", 1, 1); np = dev->of_node; - base = of_iomap(np, 0); - if (WARN_ON(!base)) - return -ENOMEM; + base = devm_platform_ioremap_resource(pdev, 0); + if (WARN_ON(IS_ERR(base))) + return PTR_ERR(base); /* CORE */ clks[IMX8MQ_CLK_A53_SRC] = imx_clk_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mq_a53_sels, ARRAY_SIZE(imx8mq_a53_sels)); @@ -442,6 +452,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_CLK_UART4] = imx8m_clk_composite("uart4", imx8mq_uart4_sels, base + 0xb080); clks[IMX8MQ_CLK_USB_CORE_REF] = imx8m_clk_composite("usb_core_ref", imx8mq_usb_core_sels, base + 0xb100); clks[IMX8MQ_CLK_USB_PHY_REF] = imx8m_clk_composite("usb_phy_ref", imx8mq_usb_phy_sels, base + 0xb180); + clks[IMX8MQ_CLK_GIC] = imx8m_clk_composite_critical("gic", imx8mq_gic_sels, base + 0xb200); clks[IMX8MQ_CLK_ECSPI1] = imx8m_clk_composite("ecspi1", imx8mq_ecspi1_sels, base + 0xb280); clks[IMX8MQ_CLK_ECSPI2] = imx8m_clk_composite("ecspi2", imx8mq_ecspi2_sels, base + 0xb300); clks[IMX8MQ_CLK_PWM1] = imx8m_clk_composite("pwm1", imx8mq_pwm1_sels, base + 0xb380); @@ -507,6 +518,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_CLK_SAI5_IPG] = imx_clk_gate2_shared2("sai5_ipg_clk", "ipg_audio_root", base + 0x4370, 0, &share_count_sai5); clks[IMX8MQ_CLK_SAI6_ROOT] = imx_clk_gate2_shared2("sai6_root_clk", "sai6", base + 0x4380, 0, &share_count_sai6); clks[IMX8MQ_CLK_SAI6_IPG] = imx_clk_gate2_shared2("sai6_ipg_clk", "ipg_audio_root", base + 0x4380, 0, &share_count_sai6); + clks[IMX8MQ_CLK_SNVS_ROOT] = imx_clk_gate4("snvs_root_clk", "ipg_root", base + 0x4470, 0); clks[IMX8MQ_CLK_UART1_ROOT] = imx_clk_gate4("uart1_root_clk", "uart1", base + 0x4490, 0); clks[IMX8MQ_CLK_UART2_ROOT] = imx_clk_gate4("uart2_root_clk", "uart2", base + 0x44a0, 0); clks[IMX8MQ_CLK_UART3_ROOT] = imx_clk_gate4("uart3_root_clk", "uart3", base + 0x44b0, 0); @@ -543,10 +555,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) clks[IMX8MQ_ARM_PLL_OUT], clks[IMX8MQ_SYS1_PLL_800M]); - for (i = 0; i < IMX8MQ_CLK_END; i++) - if (IS_ERR(clks[i])) - pr_err("i.MX8mq clk %u register failed with %ld\n", - i, PTR_ERR(clks[i])); + imx_check_clocks(clks, ARRAY_SIZE(clks)); clk_data.clks = clks; clk_data.clk_num = ARRAY_SIZE(clks); @@ -554,6 +563,8 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) err = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); WARN_ON(err); + imx_register_uart_clocks(uart_clks); + return err; } diff --git a/drivers/clk/imx/clk-pfd.c b/drivers/clk/imx/clk-pfd.c index fa82ddea4996..50b7c30296f7 100644 --- a/drivers/clk/imx/clk-pfd.c +++ b/drivers/clk/imx/clk-pfd.c @@ -121,12 +121,13 @@ static const struct clk_ops clk_pfd_ops = { .is_enabled = clk_pfd_is_enabled, }; -struct clk *imx_clk_pfd(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_pfd(const char *name, const char *parent_name, void __iomem *reg, u8 idx) { struct clk_pfd *pfd; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; pfd = kzalloc(sizeof(*pfd), GFP_KERNEL); if (!pfd) @@ -142,10 +143,13 @@ struct clk *imx_clk_pfd(const char *name, const char *parent_name, init.num_parents = 1; pfd->hw.init = &init; + hw = &pfd->hw; - clk = clk_register(NULL, &pfd->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(pfd); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c index 93b059608d3c..df91a8244fb4 100644 --- a/drivers/clk/imx/clk-pllv3.c +++ b/drivers/clk/imx/clk-pllv3.c @@ -410,14 +410,15 @@ static const struct clk_ops clk_pllv3_enet_ops = { .recalc_rate = clk_pllv3_enet_recalc_rate, }; -struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name, +struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name, const char *parent_name, void __iomem *base, u32 div_mask) { struct clk_pllv3 *pll; const struct clk_ops *ops; - struct clk *clk; + struct clk_hw *hw; struct clk_init_data init; + int ret; pll = kzalloc(sizeof(*pll), GFP_KERNEL); if (!pll) @@ -478,10 +479,13 @@ struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name, init.num_parents = 1; pll->hw.init = &init; + hw = &pll->hw; - clk = clk_register(NULL, &pll->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(NULL, hw); + if (ret) { kfree(pll); + return ERR_PTR(ret); + } - return clk; + return hw; } diff --git a/drivers/clk/imx/clk.c b/drivers/clk/imx/clk.c index 1efed86217f7..f628071f6605 100644 --- a/drivers/clk/imx/clk.c +++ b/drivers/clk/imx/clk.c @@ -1,14 +1,30 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/clk.h> +#include <linux/clk-provider.h> #include <linux/err.h> +#include <linux/io.h> #include <linux/of.h> #include <linux/slab.h> #include <linux/spinlock.h> #include "clk.h" +#define CCM_CCDR 0x4 +#define CCDR_MMDC_CH0_MASK BIT(17) +#define CCDR_MMDC_CH1_MASK BIT(16) + DEFINE_SPINLOCK(imx_ccm_lock); -void __init imx_check_clocks(struct clk *clks[], unsigned int count) +void __init imx_mmdc_mask_handshake(void __iomem *ccm_base, + unsigned int chn) +{ + unsigned int reg; + + reg = readl_relaxed(ccm_base + CCM_CCDR); + reg |= chn == 0 ? CCDR_MMDC_CH0_MASK : CCDR_MMDC_CH1_MASK; + writel_relaxed(reg, ccm_base + CCM_CCDR); +} + +void imx_check_clocks(struct clk *clks[], unsigned int count) { unsigned i; @@ -59,6 +75,17 @@ struct clk * __init imx_obtain_fixed_clock( return clk; } +struct clk_hw * __init imx_obtain_fixed_clock_hw( + const char *name, unsigned long rate) +{ + struct clk *clk; + + clk = imx_obtain_fixed_clock_from_dt(name); + if (IS_ERR(clk)) + clk = imx_clk_fixed(name, rate); + return __clk_get_hw(clk); +} + struct clk_hw * __init imx_obtain_fixed_clk_hw(struct device_node *np, const char *name) { @@ -97,8 +124,8 @@ void imx_cscmr1_fixup(u32 *val) return; } -static int imx_keep_uart_clocks __initdata; -static struct clk ** const *imx_uart_clocks __initdata; +static int imx_keep_uart_clocks; +static struct clk ** const *imx_uart_clocks; static int __init imx_keep_uart_clocks_param(char *str) { @@ -111,7 +138,7 @@ __setup_param("earlycon", imx_keep_uart_earlycon, __setup_param("earlyprintk", imx_keep_uart_earlyprintk, imx_keep_uart_clocks_param, 0); -void __init imx_register_uart_clocks(struct clk ** const clks[]) +void imx_register_uart_clocks(struct clk ** const clks[]) { if (imx_keep_uart_clocks) { int i; diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h index 8639a8f2153e..d94d9cb079d3 100644 --- a/drivers/clk/imx/clk.h +++ b/drivers/clk/imx/clk.h @@ -10,6 +10,8 @@ extern spinlock_t imx_ccm_lock; void imx_check_clocks(struct clk *clks[], unsigned int count); void imx_check_clk_hws(struct clk_hw *clks[], unsigned int count); void imx_register_uart_clocks(struct clk ** const clks[]); +void imx_register_uart_clocks_hws(struct clk_hw ** const hws[]); +void imx_mmdc_mask_handshake(void __iomem *ccm_base, unsigned int chn); extern void imx_cscmr1_fixup(u32 *val); @@ -48,6 +50,74 @@ struct imx_pll14xx_clk { int flags; }; +#define imx_clk_busy_divider(name, parent_name, reg, shift, width, busy_reg, busy_shift) \ + imx_clk_hw_busy_divider(name, parent_name, reg, shift, width, busy_reg, busy_shift)->clk + +#define imx_clk_busy_mux(name, reg, shift, width, busy_reg, busy_shift, parent_names, num_parents) \ + imx_clk_hw_busy_mux(name, reg, shift, width, busy_reg, busy_shift, parent_names, num_parents)->clk + +#define imx_clk_cpu(name, parent_name, div, mux, pll, step) \ + imx_clk_hw_cpu(name, parent_name, div, mux, pll, step)->clk + +#define clk_register_gate2(dev, name, parent_name, flags, reg, bit_idx, \ + cgr_val, clk_gate_flags, lock, share_count) \ + clk_hw_register_gate2(dev, name, parent_name, flags, reg, bit_idx, \ + cgr_val, clk_gate_flags, lock, share_count)->clk + +#define imx_clk_pllv3(type, name, parent_name, base, div_mask) \ + imx_clk_hw_pllv3(type, name, parent_name, base, div_mask)->clk + +#define imx_clk_pfd(name, parent_name, reg, idx) \ + imx_clk_hw_pfd(name, parent_name, reg, idx)->clk + +#define imx_clk_gate_exclusive(name, parent, reg, shift, exclusive_mask) \ + imx_clk_hw_gate_exclusive(name, parent, reg, shift, exclusive_mask)->clk + +#define imx_clk_fixup_divider(name, parent, reg, shift, width, fixup) \ + imx_clk_hw_fixup_divider(name, parent, reg, shift, width, fixup)->clk + +#define imx_clk_fixup_mux(name, reg, shift, width, parents, num_parents, fixup) \ + imx_clk_hw_fixup_mux(name, reg, shift, width, parents, num_parents, fixup)->clk + +#define imx_clk_mux_ldb(name, reg, shift, width, parents, num_parents) \ + imx_clk_hw_mux_ldb(name, reg, shift, width, parents, num_parents)->clk + +#define imx_clk_fixed_factor(name, parent, mult, div) \ + imx_clk_hw_fixed_factor(name, parent, mult, div)->clk + +#define imx_clk_divider2(name, parent, reg, shift, width) \ + imx_clk_hw_divider2(name, parent, reg, shift, width)->clk + +#define imx_clk_gate_dis(name, parent, reg, shift) \ + imx_clk_hw_gate_dis(name, parent, reg, shift)->clk + +#define imx_clk_gate_dis_flags(name, parent, reg, shift, flags) \ + imx_clk_hw_gate_dis_flags(name, parent, reg, shift, flags)->clk + +#define imx_clk_gate_flags(name, parent, reg, shift, flags) \ + imx_clk_hw_gate_flags(name, parent, reg, shift, flags)->clk + +#define imx_clk_gate2(name, parent, reg, shift) \ + imx_clk_hw_gate2(name, parent, reg, shift)->clk + +#define imx_clk_gate2_flags(name, parent, reg, shift, flags) \ + imx_clk_hw_gate2_flags(name, parent, reg, shift, flags)->clk + +#define imx_clk_gate2_shared(name, parent, reg, shift, share_count) \ + imx_clk_hw_gate2_shared(name, parent, reg, shift, share_count)->clk + +#define imx_clk_gate2_shared2(name, parent, reg, shift, share_count) \ + imx_clk_hw_gate2_shared2(name, parent, reg, shift, share_count)->clk + +#define imx_clk_gate3(name, parent, reg, shift) \ + imx_clk_hw_gate3(name, parent, reg, shift)->clk + +#define imx_clk_gate4(name, parent, reg, shift) \ + imx_clk_hw_gate4(name, parent, reg, shift)->clk + +#define imx_clk_mux(name, reg, shift, width, parents, num_parents) \ + imx_clk_hw_mux(name, reg, shift, width, parents, num_parents)->clk + struct clk *imx_clk_pll14xx(const char *name, const char *parent_name, void __iomem *base, const struct imx_pll14xx_clk *pll_clk); @@ -80,13 +150,13 @@ enum imx_pllv3_type { IMX_PLLV3_AV_IMX7, }; -struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name, +struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name, const char *parent_name, void __iomem *base, u32 div_mask); struct clk_hw *imx_clk_pllv4(const char *name, const char *parent_name, void __iomem *base); -struct clk *clk_register_gate2(struct device *dev, const char *name, +struct clk_hw *clk_hw_register_gate2(struct device *dev, const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 bit_idx, u8 cgr_val, u8 clk_gate_flags, spinlock_t *lock, @@ -95,23 +165,26 @@ struct clk *clk_register_gate2(struct device *dev, const char *name, struct clk * imx_obtain_fixed_clock( const char *name, unsigned long rate); +struct clk_hw *imx_obtain_fixed_clock_hw( + const char *name, unsigned long rate); + struct clk_hw *imx_obtain_fixed_clk_hw(struct device_node *np, const char *name); -struct clk *imx_clk_gate_exclusive(const char *name, const char *parent, +struct clk_hw *imx_clk_hw_gate_exclusive(const char *name, const char *parent, void __iomem *reg, u8 shift, u32 exclusive_mask); -struct clk *imx_clk_pfd(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_pfd(const char *name, const char *parent_name, void __iomem *reg, u8 idx); struct clk_hw *imx_clk_pfdv2(const char *name, const char *parent_name, void __iomem *reg, u8 idx); -struct clk *imx_clk_busy_divider(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_busy_divider(const char *name, const char *parent_name, void __iomem *reg, u8 shift, u8 width, void __iomem *busy_reg, u8 busy_shift); -struct clk *imx_clk_busy_mux(const char *name, void __iomem *reg, u8 shift, +struct clk_hw *imx_clk_hw_busy_mux(const char *name, void __iomem *reg, u8 shift, u8 width, void __iomem *busy_reg, u8 busy_shift, const char * const *parent_names, int num_parents); @@ -121,11 +194,11 @@ struct clk_hw *imx7ulp_clk_composite(const char *name, bool rate_present, bool gate_present, void __iomem *reg); -struct clk *imx_clk_fixup_divider(const char *name, const char *parent, +struct clk_hw *imx_clk_hw_fixup_divider(const char *name, const char *parent, void __iomem *reg, u8 shift, u8 width, void (*fixup)(u32 *val)); -struct clk *imx_clk_fixup_mux(const char *name, void __iomem *reg, +struct clk_hw *imx_clk_hw_fixup_mux(const char *name, void __iomem *reg, u8 shift, u8 width, const char * const *parents, int num_parents, void (*fixup)(u32 *val)); @@ -139,19 +212,19 @@ static inline struct clk_hw *imx_clk_hw_fixed(const char *name, int rate) return clk_hw_register_fixed_rate(NULL, name, NULL, 0, rate); } -static inline struct clk *imx_clk_mux_ldb(const char *name, void __iomem *reg, +static inline struct clk_hw *imx_clk_hw_mux_ldb(const char *name, void __iomem *reg, u8 shift, u8 width, const char * const *parents, int num_parents) { - return clk_register_mux(NULL, name, parents, num_parents, + return clk_hw_register_mux(NULL, name, parents, num_parents, CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, reg, shift, width, CLK_MUX_READ_ONLY, &imx_ccm_lock); } -static inline struct clk *imx_clk_fixed_factor(const char *name, +static inline struct clk_hw *imx_clk_hw_fixed_factor(const char *name, const char *parent, unsigned int mult, unsigned int div) { - return clk_register_fixed_factor(NULL, name, parent, + return clk_hw_register_fixed_factor(NULL, name, parent, CLK_SET_RATE_PARENT, mult, div); } @@ -188,10 +261,10 @@ static inline struct clk_hw *imx_clk_hw_divider_flags(const char *name, reg, shift, width, 0, &imx_ccm_lock); } -static inline struct clk *imx_clk_divider2(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_divider2(const char *name, const char *parent, void __iomem *reg, u8 shift, u8 width) { - return clk_register_divider(NULL, name, parent, + return clk_hw_register_divider(NULL, name, parent, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, reg, shift, width, 0, &imx_ccm_lock); } @@ -212,10 +285,10 @@ static inline struct clk *imx_clk_gate(const char *name, const char *parent, shift, 0, &imx_ccm_lock); } -static inline struct clk *imx_clk_gate_flags(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_gate_flags(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned long flags) { - return clk_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg, + return clk_hw_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg, shift, 0, &imx_ccm_lock); } @@ -226,47 +299,47 @@ static inline struct clk_hw *imx_clk_hw_gate(const char *name, const char *paren shift, 0, &imx_ccm_lock); } -static inline struct clk *imx_clk_gate_dis(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_gate_dis(const char *name, const char *parent, void __iomem *reg, u8 shift) { - return clk_register_gate(NULL, name, parent, CLK_SET_RATE_PARENT, reg, + return clk_hw_register_gate(NULL, name, parent, CLK_SET_RATE_PARENT, reg, shift, CLK_GATE_SET_TO_DISABLE, &imx_ccm_lock); } -static inline struct clk *imx_clk_gate_dis_flags(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_gate_dis_flags(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned long flags) { - return clk_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg, + return clk_hw_register_gate(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg, shift, CLK_GATE_SET_TO_DISABLE, &imx_ccm_lock); } -static inline struct clk *imx_clk_gate2(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_gate2(const char *name, const char *parent, void __iomem *reg, u8 shift) { - return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, + return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, shift, 0x3, 0, &imx_ccm_lock, NULL); } -static inline struct clk *imx_clk_gate2_flags(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_gate2_flags(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned long flags) { - return clk_register_gate2(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg, + return clk_hw_register_gate2(NULL, name, parent, flags | CLK_SET_RATE_PARENT, reg, shift, 0x3, 0, &imx_ccm_lock, NULL); } -static inline struct clk *imx_clk_gate2_shared(const char *name, +static inline struct clk_hw *imx_clk_hw_gate2_shared(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned int *share_count) { - return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, + return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, shift, 0x3, 0, &imx_ccm_lock, share_count); } -static inline struct clk *imx_clk_gate2_shared2(const char *name, +static inline struct clk_hw *imx_clk_hw_gate2_shared2(const char *name, const char *parent, void __iomem *reg, u8 shift, unsigned int *share_count) { - return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT | + return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, reg, shift, 0x3, 0, &imx_ccm_lock, share_count); } @@ -278,10 +351,10 @@ static inline struct clk *imx_clk_gate2_cgr(const char *name, shift, cgr_val, 0, &imx_ccm_lock, NULL); } -static inline struct clk *imx_clk_gate3(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_gate3(const char *name, const char *parent, void __iomem *reg, u8 shift) { - return clk_register_gate(NULL, name, parent, + return clk_hw_register_gate(NULL, name, parent, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, reg, shift, 0, &imx_ccm_lock); } @@ -295,10 +368,10 @@ static inline struct clk *imx_clk_gate3_flags(const char *name, reg, shift, 0, &imx_ccm_lock); } -static inline struct clk *imx_clk_gate4(const char *name, const char *parent, +static inline struct clk_hw *imx_clk_hw_gate4(const char *name, const char *parent, void __iomem *reg, u8 shift) { - return clk_register_gate2(NULL, name, parent, + return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, reg, shift, 0x3, 0, &imx_ccm_lock, NULL); } @@ -312,11 +385,11 @@ static inline struct clk *imx_clk_gate4_flags(const char *name, reg, shift, 0x3, 0, &imx_ccm_lock, NULL); } -static inline struct clk *imx_clk_mux(const char *name, void __iomem *reg, +static inline struct clk_hw *imx_clk_hw_mux(const char *name, void __iomem *reg, u8 shift, u8 width, const char * const *parents, int num_parents) { - return clk_register_mux(NULL, name, parents, num_parents, + return clk_hw_register_mux(NULL, name, parents, num_parents, CLK_SET_RATE_NO_REPARENT, reg, shift, width, 0, &imx_ccm_lock); } @@ -373,7 +446,7 @@ static inline struct clk_hw *imx_clk_hw_mux_flags(const char *name, reg, shift, width, 0, &imx_ccm_lock); } -struct clk *imx_clk_cpu(const char *name, const char *parent_name, +struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name, struct clk *div, struct clk *mux, struct clk *pll, struct clk *step); diff --git a/drivers/clk/ingenic/Makefile b/drivers/clk/ingenic/Makefile index ab58a6a862a5..250570a809d3 100644 --- a/drivers/clk/ingenic/Makefile +++ b/drivers/clk/ingenic/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_INGENIC_CGU_COMMON) += cgu.o +obj-$(CONFIG_INGENIC_CGU_COMMON) += cgu.o pm.o obj-$(CONFIG_INGENIC_CGU_JZ4740) += jz4740-cgu.o obj-$(CONFIG_INGENIC_CGU_JZ4725B) += jz4725b-cgu.o obj-$(CONFIG_INGENIC_CGU_JZ4770) += jz4770-cgu.o diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c index 92c331427513..6e963031cd87 100644 --- a/drivers/clk/ingenic/cgu.c +++ b/drivers/clk/ingenic/cgu.c @@ -375,8 +375,11 @@ ingenic_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) div_reg = readl(cgu->base + clk_info->div.reg); div = (div_reg >> clk_info->div.shift) & GENMASK(clk_info->div.bits - 1, 0); - div += 1; - div *= clk_info->div.div; + + if (clk_info->div.div_table) + div = clk_info->div.div_table[div]; + else + div = (div + 1) * clk_info->div.div; rate /= div; } else if (clk_info->type & CGU_CLK_FIXDIV) { @@ -386,16 +389,37 @@ ingenic_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) return rate; } +static unsigned int +ingenic_clk_calc_hw_div(const struct ingenic_cgu_clk_info *clk_info, + unsigned int div) +{ + unsigned int i; + + for (i = 0; i < (1 << clk_info->div.bits) + && clk_info->div.div_table[i]; i++) { + if (clk_info->div.div_table[i] >= div) + return i; + } + + return i - 1; +} + static unsigned ingenic_clk_calc_div(const struct ingenic_cgu_clk_info *clk_info, unsigned long parent_rate, unsigned long req_rate) { - unsigned div; + unsigned int div, hw_div; /* calculate the divide */ div = DIV_ROUND_UP(parent_rate, req_rate); - /* and impose hardware constraints */ + if (clk_info->div.div_table) { + hw_div = ingenic_clk_calc_hw_div(clk_info, div); + + return clk_info->div.div_table[hw_div]; + } + + /* Impose hardware constraints */ div = min_t(unsigned, div, 1 << clk_info->div.bits); div = max_t(unsigned, div, 1); @@ -438,7 +462,7 @@ ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate, const struct ingenic_cgu_clk_info *clk_info; const unsigned timeout = 100; unsigned long rate, flags; - unsigned div, i; + unsigned int hw_div, div, i; u32 reg, mask; int ret = 0; @@ -451,13 +475,18 @@ ingenic_clk_set_rate(struct clk_hw *hw, unsigned long req_rate, if (rate != req_rate) return -EINVAL; + if (clk_info->div.div_table) + hw_div = ingenic_clk_calc_hw_div(clk_info, div); + else + hw_div = ((div / clk_info->div.div) - 1); + spin_lock_irqsave(&cgu->lock, flags); reg = readl(cgu->base + clk_info->div.reg); /* update the divide */ mask = GENMASK(clk_info->div.bits - 1, 0); reg &= ~(mask << clk_info->div.shift); - reg |= ((div / clk_info->div.div) - 1) << clk_info->div.shift; + reg |= hw_div << clk_info->div.shift; /* clear the stop bit */ if (clk_info->div.stop_bit != -1) diff --git a/drivers/clk/ingenic/cgu.h b/drivers/clk/ingenic/cgu.h index bfbcf6db437d..0dc8004079ee 100644 --- a/drivers/clk/ingenic/cgu.h +++ b/drivers/clk/ingenic/cgu.h @@ -10,6 +10,7 @@ #define __DRIVERS_CLK_INGENIC_CGU_H__ #include <linux/bitops.h> +#include <linux/clk-provider.h> #include <linux/of.h> #include <linux/spinlock.h> @@ -79,6 +80,8 @@ struct ingenic_cgu_mux_info { * isn't one * @busy_bit: the index of the busy bit within reg, or -1 if there isn't one * @stop_bit: the index of the stop bit within reg, or -1 if there isn't one + * @div_table: optional table to map the value read from the register to the + * actual divider value */ struct ingenic_cgu_div_info { unsigned reg; @@ -88,6 +91,7 @@ struct ingenic_cgu_div_info { s8 ce_bit; s8 busy_bit; s8 stop_bit; + const u8 *div_table; }; /** diff --git a/drivers/clk/ingenic/jz4725b-cgu.c b/drivers/clk/ingenic/jz4725b-cgu.c index 8901ea0295b7..2642d36d1e2c 100644 --- a/drivers/clk/ingenic/jz4725b-cgu.c +++ b/drivers/clk/ingenic/jz4725b-cgu.c @@ -11,6 +11,7 @@ #include <linux/of.h> #include <dt-bindings/clock/jz4725b-cgu.h> #include "cgu.h" +#include "pm.h" /* CGU register offsets */ #define CGU_REG_CPCCR 0x00 @@ -33,6 +34,14 @@ static const s8 pll_od_encoding[4] = { 0x0, 0x1, -1, 0x3, }; +static const u8 jz4725b_cgu_cpccr_div_table[] = { + 1, 2, 3, 4, 6, 8, +}; + +static const u8 jz4725b_cgu_pll_half_div_table[] = { + 2, 1, +}; + static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = { /* External clocks */ @@ -66,37 +75,55 @@ static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = { [JZ4725B_CLK_PLL_HALF] = { "pll half", CGU_CLK_DIV, .parents = { JZ4725B_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 21, 1, 1, -1, -1, -1 }, + .div = { + CGU_REG_CPCCR, 21, 1, 1, -1, -1, -1, + jz4725b_cgu_pll_half_div_table, + }, }, [JZ4725B_CLK_CCLK] = { "cclk", CGU_CLK_DIV, .parents = { JZ4725B_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1, + jz4725b_cgu_cpccr_div_table, + }, }, [JZ4725B_CLK_HCLK] = { "hclk", CGU_CLK_DIV, .parents = { JZ4725B_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1, + jz4725b_cgu_cpccr_div_table, + }, }, [JZ4725B_CLK_PCLK] = { "pclk", CGU_CLK_DIV, .parents = { JZ4725B_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1, + jz4725b_cgu_cpccr_div_table, + }, }, [JZ4725B_CLK_MCLK] = { "mclk", CGU_CLK_DIV, .parents = { JZ4725B_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1, + jz4725b_cgu_cpccr_div_table, + }, }, [JZ4725B_CLK_IPU] = { "ipu", CGU_CLK_DIV | CGU_CLK_GATE, .parents = { JZ4725B_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1, + jz4725b_cgu_cpccr_div_table, + }, .gate = { CGU_REG_CLKGR, 13 }, }, @@ -227,5 +254,7 @@ static void __init jz4725b_cgu_init(struct device_node *np) retval = ingenic_cgu_register_clocks(cgu); if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); + + ingenic_cgu_register_syscore_ops(cgu); } CLK_OF_DECLARE(jz4725b_cgu, "ingenic,jz4725b-cgu", jz4725b_cgu_init); diff --git a/drivers/clk/ingenic/jz4740-cgu.c b/drivers/clk/ingenic/jz4740-cgu.c index c77f4e1506dc..4c0a20949c2c 100644 --- a/drivers/clk/ingenic/jz4740-cgu.c +++ b/drivers/clk/ingenic/jz4740-cgu.c @@ -11,8 +11,8 @@ #include <linux/io.h> #include <linux/of.h> #include <dt-bindings/clock/jz4740-cgu.h> -#include <asm/mach-jz4740/clock.h> #include "cgu.h" +#include "pm.h" /* CGU register offsets */ #define CGU_REG_CPCCR 0x00 @@ -49,6 +49,10 @@ static const s8 pll_od_encoding[4] = { 0x0, 0x1, -1, 0x3, }; +static const u8 jz4740_cgu_cpccr_div_table[] = { + 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, +}; + static const struct ingenic_cgu_clk_info jz4740_cgu_clocks[] = { /* External clocks */ @@ -88,31 +92,46 @@ static const struct ingenic_cgu_clk_info jz4740_cgu_clocks[] = { [JZ4740_CLK_CCLK] = { "cclk", CGU_CLK_DIV, .parents = { JZ4740_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1, + jz4740_cgu_cpccr_div_table, + }, }, [JZ4740_CLK_HCLK] = { "hclk", CGU_CLK_DIV, .parents = { JZ4740_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1, + jz4740_cgu_cpccr_div_table, + }, }, [JZ4740_CLK_PCLK] = { "pclk", CGU_CLK_DIV, .parents = { JZ4740_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1, + jz4740_cgu_cpccr_div_table, + }, }, [JZ4740_CLK_MCLK] = { "mclk", CGU_CLK_DIV, .parents = { JZ4740_CLK_PLL, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1, + jz4740_cgu_cpccr_div_table, + }, }, [JZ4740_CLK_LCD] = { "lcd", CGU_CLK_DIV | CGU_CLK_GATE, .parents = { JZ4740_CLK_PLL_HALF, -1, -1, -1 }, - .div = { CGU_REG_CPCCR, 16, 1, 5, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 16, 1, 5, 22, -1, -1, + jz4740_cgu_cpccr_div_table, + }, .gate = { CGU_REG_CLKGR, 10 }, }, @@ -219,77 +238,7 @@ static void __init jz4740_cgu_init(struct device_node *np) retval = ingenic_cgu_register_clocks(cgu); if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); -} -CLK_OF_DECLARE(jz4740_cgu, "ingenic,jz4740-cgu", jz4740_cgu_init); - -void jz4740_clock_set_wait_mode(enum jz4740_wait_mode mode) -{ - uint32_t lcr = readl(cgu->base + CGU_REG_LCR); - - switch (mode) { - case JZ4740_WAIT_MODE_IDLE: - lcr &= ~LCR_SLEEP; - break; - - case JZ4740_WAIT_MODE_SLEEP: - lcr |= LCR_SLEEP; - break; - } - - writel(lcr, cgu->base + CGU_REG_LCR); -} -void jz4740_clock_udc_disable_auto_suspend(void) -{ - uint32_t clkgr = readl(cgu->base + CGU_REG_CLKGR); - - clkgr &= ~CLKGR_UDC; - writel(clkgr, cgu->base + CGU_REG_CLKGR); -} -EXPORT_SYMBOL_GPL(jz4740_clock_udc_disable_auto_suspend); - -void jz4740_clock_udc_enable_auto_suspend(void) -{ - uint32_t clkgr = readl(cgu->base + CGU_REG_CLKGR); - - clkgr |= CLKGR_UDC; - writel(clkgr, cgu->base + CGU_REG_CLKGR); -} -EXPORT_SYMBOL_GPL(jz4740_clock_udc_enable_auto_suspend); - -#define JZ_CLOCK_GATE_UART0 BIT(0) -#define JZ_CLOCK_GATE_TCU BIT(1) -#define JZ_CLOCK_GATE_DMAC BIT(12) - -void jz4740_clock_suspend(void) -{ - uint32_t clkgr, cppcr; - - clkgr = readl(cgu->base + CGU_REG_CLKGR); - clkgr |= JZ_CLOCK_GATE_TCU | JZ_CLOCK_GATE_DMAC | JZ_CLOCK_GATE_UART0; - writel(clkgr, cgu->base + CGU_REG_CLKGR); - - cppcr = readl(cgu->base + CGU_REG_CPPCR); - cppcr &= ~BIT(jz4740_cgu_clocks[JZ4740_CLK_PLL].pll.enable_bit); - writel(cppcr, cgu->base + CGU_REG_CPPCR); -} - -void jz4740_clock_resume(void) -{ - uint32_t clkgr, cppcr, stable; - - cppcr = readl(cgu->base + CGU_REG_CPPCR); - cppcr |= BIT(jz4740_cgu_clocks[JZ4740_CLK_PLL].pll.enable_bit); - writel(cppcr, cgu->base + CGU_REG_CPPCR); - - stable = BIT(jz4740_cgu_clocks[JZ4740_CLK_PLL].pll.stable_bit); - do { - cppcr = readl(cgu->base + CGU_REG_CPPCR); - } while (!(cppcr & stable)); - - clkgr = readl(cgu->base + CGU_REG_CLKGR); - clkgr &= ~JZ_CLOCK_GATE_TCU; - clkgr &= ~JZ_CLOCK_GATE_DMAC; - clkgr &= ~JZ_CLOCK_GATE_UART0; - writel(clkgr, cgu->base + CGU_REG_CLKGR); + ingenic_cgu_register_syscore_ops(cgu); } +CLK_OF_DECLARE(jz4740_cgu, "ingenic,jz4740-cgu", jz4740_cgu_init); diff --git a/drivers/clk/ingenic/jz4770-cgu.c b/drivers/clk/ingenic/jz4770-cgu.c index dfce740c25a8..eebc1bea3841 100644 --- a/drivers/clk/ingenic/jz4770-cgu.c +++ b/drivers/clk/ingenic/jz4770-cgu.c @@ -9,9 +9,9 @@ #include <linux/delay.h> #include <linux/io.h> #include <linux/of.h> -#include <linux/syscore_ops.h> #include <dt-bindings/clock/jz4770-cgu.h> #include "cgu.h" +#include "pm.h" /* * CPM registers offset address definition @@ -38,9 +38,6 @@ #define CGU_REG_MSC2CDR 0xA8 #define CGU_REG_BCHCDR 0xAC -/* bits within the LCR register */ -#define LCR_LPM BIT(0) /* Low Power Mode */ - /* bits within the OPCR register */ #define OPCR_SPENDH BIT(5) /* UHC PHY suspend */ @@ -87,6 +84,10 @@ static const s8 pll_od_encoding[8] = { 0x0, 0x1, -1, 0x2, -1, -1, -1, 0x3, }; +static const u8 jz4770_cgu_cpccr_div_table[] = { + 1, 2, 3, 4, 6, 8, 12, +}; + static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = { /* External clocks */ @@ -144,34 +145,52 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = { [JZ4770_CLK_CCLK] = { "cclk", CGU_CLK_DIV, .parents = { JZ4770_CLK_PLL0, }, - .div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1, + jz4770_cgu_cpccr_div_table, + }, }, [JZ4770_CLK_H0CLK] = { "h0clk", CGU_CLK_DIV, .parents = { JZ4770_CLK_PLL0, }, - .div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1, + jz4770_cgu_cpccr_div_table, + }, }, [JZ4770_CLK_H1CLK] = { "h1clk", CGU_CLK_DIV | CGU_CLK_GATE, .parents = { JZ4770_CLK_PLL0, }, - .div = { CGU_REG_CPCCR, 24, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 24, 1, 4, 22, -1, -1, + jz4770_cgu_cpccr_div_table, + }, .gate = { CGU_REG_CLKGR1, 7 }, }, [JZ4770_CLK_H2CLK] = { "h2clk", CGU_CLK_DIV, .parents = { JZ4770_CLK_PLL0, }, - .div = { CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1, + jz4770_cgu_cpccr_div_table, + }, }, [JZ4770_CLK_C1CLK] = { "c1clk", CGU_CLK_DIV | CGU_CLK_GATE, .parents = { JZ4770_CLK_PLL0, }, - .div = { CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1, + jz4770_cgu_cpccr_div_table, + }, .gate = { CGU_REG_OPCR, 31, true }, // disable CCLK stop on idle }, [JZ4770_CLK_PCLK] = { "pclk", CGU_CLK_DIV, .parents = { JZ4770_CLK_PLL0, }, - .div = { CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1 }, + .div = { + CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1, + jz4770_cgu_cpccr_div_table, + }, }, /* Those divided clocks can connect to PLL0 or PLL1 */ @@ -407,30 +426,6 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = { }, }; -#if IS_ENABLED(CONFIG_PM_SLEEP) -static int jz4770_cgu_pm_suspend(void) -{ - u32 val; - - val = readl(cgu->base + CGU_REG_LCR); - writel(val | LCR_LPM, cgu->base + CGU_REG_LCR); - return 0; -} - -static void jz4770_cgu_pm_resume(void) -{ - u32 val; - - val = readl(cgu->base + CGU_REG_LCR); - writel(val & ~LCR_LPM, cgu->base + CGU_REG_LCR); -} - -static struct syscore_ops jz4770_cgu_pm_ops = { - .suspend = jz4770_cgu_pm_suspend, - .resume = jz4770_cgu_pm_resume, -}; -#endif /* CONFIG_PM_SLEEP */ - static void __init jz4770_cgu_init(struct device_node *np) { int retval; @@ -444,9 +439,7 @@ static void __init jz4770_cgu_init(struct device_node *np) if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); -#if IS_ENABLED(CONFIG_PM_SLEEP) - register_syscore_ops(&jz4770_cgu_pm_ops); -#endif + ingenic_cgu_register_syscore_ops(cgu); } /* We only probe via devicetree, no need for a platform driver */ diff --git a/drivers/clk/ingenic/jz4780-cgu.c b/drivers/clk/ingenic/jz4780-cgu.c index 2464fc4032af..8c67f89df25e 100644 --- a/drivers/clk/ingenic/jz4780-cgu.c +++ b/drivers/clk/ingenic/jz4780-cgu.c @@ -12,6 +12,7 @@ #include <linux/of.h> #include <dt-bindings/clock/jz4780-cgu.h> #include "cgu.h" +#include "pm.h" /* CGU register offsets */ #define CGU_REG_CLOCKCONTROL 0x00 @@ -721,5 +722,7 @@ static void __init jz4780_cgu_init(struct device_node *np) pr_err("%s: failed to register CGU Clocks\n", __func__); return; } + + ingenic_cgu_register_syscore_ops(cgu); } CLK_OF_DECLARE(jz4780_cgu, "ingenic,jz4780-cgu", jz4780_cgu_init); diff --git a/drivers/clk/ingenic/pm.c b/drivers/clk/ingenic/pm.c new file mode 100644 index 000000000000..341752b640d2 --- /dev/null +++ b/drivers/clk/ingenic/pm.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 Paul Cercueil <paul@crapouillou.net> + */ + +#include "cgu.h" +#include "pm.h" + +#include <linux/io.h> +#include <linux/syscore_ops.h> + +#define CGU_REG_LCR 0x04 + +#define LCR_LOW_POWER_MODE BIT(0) + +static void __iomem * __maybe_unused ingenic_cgu_base; + +static int __maybe_unused ingenic_cgu_pm_suspend(void) +{ + u32 val = readl(ingenic_cgu_base + CGU_REG_LCR); + + writel(val | LCR_LOW_POWER_MODE, ingenic_cgu_base + CGU_REG_LCR); + + return 0; +} + +static void __maybe_unused ingenic_cgu_pm_resume(void) +{ + u32 val = readl(ingenic_cgu_base + CGU_REG_LCR); + + writel(val & ~LCR_LOW_POWER_MODE, ingenic_cgu_base + CGU_REG_LCR); +} + +static struct syscore_ops __maybe_unused ingenic_cgu_pm_ops = { + .suspend = ingenic_cgu_pm_suspend, + .resume = ingenic_cgu_pm_resume, +}; + +void ingenic_cgu_register_syscore_ops(struct ingenic_cgu *cgu) +{ + if (IS_ENABLED(CONFIG_PM_SLEEP)) { + ingenic_cgu_base = cgu->base; + register_syscore_ops(&ingenic_cgu_pm_ops); + } +} diff --git a/drivers/clk/ingenic/pm.h b/drivers/clk/ingenic/pm.h new file mode 100644 index 000000000000..fa7540407b6b --- /dev/null +++ b/drivers/clk/ingenic/pm.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 Paul Cercueil <paul@crapouillou.net> + */ +#ifndef DRIVERS_CLK_INGENIC_PM_H +#define DRIVERS_CLK_INGENIC_PM_H + +struct ingenic_cgu; + +void ingenic_cgu_register_syscore_ops(struct ingenic_cgu *cgu); + +#endif /* DRIVERS_CLK_INGENIC_PM_H */ diff --git a/drivers/clk/keystone/Kconfig b/drivers/clk/keystone/Kconfig index 0ca63014718a..38aeefb1e808 100644 --- a/drivers/clk/keystone/Kconfig +++ b/drivers/clk/keystone/Kconfig @@ -15,3 +15,14 @@ config TI_SCI_CLK This adds the clock driver support over TI System Control Interface. If you wish to use clock resources from the PMMC firmware, say Y. Otherwise, say N. + +config TI_SCI_CLK_PROBE_FROM_FW + bool "Probe available clocks from firmware" + depends on TI_SCI_CLK + default n + help + Forces the TI SCI clock driver to probe available clocks from the + firmware. By default, only the used clocks are probed from DT. + This is mostly only useful for debugging purposes, and will + increase the boot time of the device. If you want the clocks probed + from firmware, say Y. Otherwise, say N. diff --git a/drivers/clk/keystone/sci-clk.c b/drivers/clk/keystone/sci-clk.c index 4cb70bed89a9..7edf8c8432b6 100644 --- a/drivers/clk/keystone/sci-clk.c +++ b/drivers/clk/keystone/sci-clk.c @@ -23,6 +23,7 @@ #include <linux/slab.h> #include <linux/soc/ti/ti_sci_protocol.h> #include <linux/bsearch.h> +#include <linux/list_sort.h> #define SCI_CLK_SSC_ENABLE BIT(0) #define SCI_CLK_ALLOW_FREQ_CHANGE BIT(1) @@ -52,14 +53,16 @@ struct sci_clk_provider { * @num_parents: Number of parents for this clock * @provider: Master clock provider * @flags: Flags for the clock + * @node: Link for handling clocks probed via DT */ struct sci_clk { struct clk_hw hw; u16 dev_id; - u8 clk_id; - u8 num_parents; + u32 clk_id; + u32 num_parents; struct sci_clk_provider *provider; u8 flags; + struct list_head node; }; #define to_sci_clk(_hw) container_of(_hw, struct sci_clk, hw) @@ -218,11 +221,11 @@ static int sci_clk_set_rate(struct clk_hw *hw, unsigned long rate, static u8 sci_clk_get_parent(struct clk_hw *hw) { struct sci_clk *clk = to_sci_clk(hw); - u8 parent_id; + u32 parent_id = 0; int ret; ret = clk->provider->ops->get_parent(clk->provider->sci, clk->dev_id, - clk->clk_id, &parent_id); + clk->clk_id, (void *)&parent_id); if (ret) { dev_err(clk->provider->dev, "get-parent failed for dev=%d, clk=%d, ret=%d\n", @@ -230,7 +233,9 @@ static u8 sci_clk_get_parent(struct clk_hw *hw) return 0; } - return parent_id - clk->clk_id - 1; + parent_id = parent_id - clk->clk_id - 1; + + return (u8)parent_id; } /** @@ -280,8 +285,8 @@ static int _sci_clk_build(struct sci_clk_provider *provider, int i; int ret = 0; - name = kasprintf(GFP_KERNEL, "%s:%d:%d", dev_name(provider->dev), - sci_clk->dev_id, sci_clk->clk_id); + name = kasprintf(GFP_KERNEL, "clk:%d:%d", sci_clk->dev_id, + sci_clk->clk_id); init.name = name; @@ -306,8 +311,7 @@ static int _sci_clk_build(struct sci_clk_provider *provider, for (i = 0; i < sci_clk->num_parents; i++) { char *parent_name; - parent_name = kasprintf(GFP_KERNEL, "%s:%d:%d", - dev_name(provider->dev), + parent_name = kasprintf(GFP_KERNEL, "clk:%d:%d", sci_clk->dev_id, sci_clk->clk_id + 1 + i); if (!parent_name) { @@ -404,22 +408,9 @@ static const struct of_device_id ti_sci_clk_of_match[] = { }; MODULE_DEVICE_TABLE(of, ti_sci_clk_of_match); -/** - * ti_sci_clk_probe - Probe function for the TI SCI clock driver - * @pdev: platform device pointer to be probed - * - * Probes the TI SCI clock device. Allocates a new clock provider - * and registers this to the common clock framework. Also applies - * any required flags to the identified clocks via clock lists - * supplied from DT. Returns 0 for success, negative error value - * for failure. - */ -static int ti_sci_clk_probe(struct platform_device *pdev) +#ifdef CONFIG_TI_SCI_CLK_PROBE_FROM_FW +static int ti_sci_scan_clocks_from_fw(struct sci_clk_provider *provider) { - struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; - struct sci_clk_provider *provider; - const struct ti_sci_handle *handle; int ret; int num_clks = 0; struct sci_clk **clks = NULL; @@ -428,24 +419,14 @@ static int ti_sci_clk_probe(struct platform_device *pdev) int max_clks = 0; int clk_id = 0; int dev_id = 0; - u8 num_parents; + u32 num_parents = 0; int gap_size = 0; - - handle = devm_ti_sci_get_handle(dev); - if (IS_ERR(handle)) - return PTR_ERR(handle); - - provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL); - if (!provider) - return -ENOMEM; - - provider->sci = handle; - provider->ops = &handle->ops.clk_ops; - provider->dev = dev; + struct device *dev = provider->dev; while (1) { ret = provider->ops->get_num_parents(provider->sci, dev_id, - clk_id, &num_parents); + clk_id, + (void *)&num_parents); if (ret) { gap_size++; if (!clk_id) { @@ -502,6 +483,188 @@ static int ti_sci_clk_probe(struct platform_device *pdev) devm_kfree(dev, clks); + return 0; +} + +#else + +static int _cmp_sci_clk_list(void *priv, struct list_head *a, + struct list_head *b) +{ + struct sci_clk *ca = container_of(a, struct sci_clk, node); + struct sci_clk *cb = container_of(b, struct sci_clk, node); + + return _cmp_sci_clk(ca, &cb); +} + +static int ti_sci_scan_clocks_from_dt(struct sci_clk_provider *provider) +{ + struct device *dev = provider->dev; + struct device_node *np = NULL; + int ret; + int index; + struct of_phandle_args args; + struct list_head clks; + struct sci_clk *sci_clk, *prev; + int num_clks = 0; + int num_parents; + int clk_id; + const char * const clk_names[] = { + "clocks", "assigned-clocks", "assigned-clock-parents", NULL + }; + const char * const *clk_name; + + INIT_LIST_HEAD(&clks); + + clk_name = clk_names; + + while (*clk_name) { + np = of_find_node_with_property(np, *clk_name); + if (!np) { + clk_name++; + break; + } + + if (!of_device_is_available(np)) + continue; + + index = 0; + + do { + ret = of_parse_phandle_with_args(np, *clk_name, + "#clock-cells", index, + &args); + if (ret) + break; + + if (args.args_count == 2 && args.np == dev->of_node) { + sci_clk = devm_kzalloc(dev, sizeof(*sci_clk), + GFP_KERNEL); + if (!sci_clk) + return -ENOMEM; + + sci_clk->dev_id = args.args[0]; + sci_clk->clk_id = args.args[1]; + sci_clk->provider = provider; + provider->ops->get_num_parents(provider->sci, + sci_clk->dev_id, + sci_clk->clk_id, + (void *)&sci_clk->num_parents); + list_add_tail(&sci_clk->node, &clks); + + num_clks++; + + num_parents = sci_clk->num_parents; + if (num_parents == 1) + num_parents = 0; + + /* + * Linux kernel has inherent limitation + * of 255 clock parents at the moment. + * Right now, it is not expected that + * any mux clock from sci-clk driver + * would exceed that limit either, but + * the ABI basically provides that + * possibility. Print out a warning if + * this happens for any clock. + */ + if (num_parents >= 255) { + dev_warn(dev, "too many parents for dev=%d, clk=%d (%d), cropping to 255.\n", + sci_clk->dev_id, + sci_clk->clk_id, num_parents); + num_parents = 255; + } + + clk_id = args.args[1] + 1; + + while (num_parents--) { + sci_clk = devm_kzalloc(dev, + sizeof(*sci_clk), + GFP_KERNEL); + if (!sci_clk) + return -ENOMEM; + sci_clk->dev_id = args.args[0]; + sci_clk->clk_id = clk_id++; + sci_clk->provider = provider; + list_add_tail(&sci_clk->node, &clks); + + num_clks++; + } + } + + index++; + } while (args.np); + } + + list_sort(NULL, &clks, _cmp_sci_clk_list); + + provider->clocks = devm_kmalloc_array(dev, num_clks, sizeof(sci_clk), + GFP_KERNEL); + if (!provider->clocks) + return -ENOMEM; + + num_clks = 0; + prev = NULL; + + list_for_each_entry(sci_clk, &clks, node) { + if (prev && prev->dev_id == sci_clk->dev_id && + prev->clk_id == sci_clk->clk_id) + continue; + + provider->clocks[num_clks++] = sci_clk; + prev = sci_clk; + } + + provider->num_clocks = num_clks; + + return 0; +} +#endif + +/** + * ti_sci_clk_probe - Probe function for the TI SCI clock driver + * @pdev: platform device pointer to be probed + * + * Probes the TI SCI clock device. Allocates a new clock provider + * and registers this to the common clock framework. Also applies + * any required flags to the identified clocks via clock lists + * supplied from DT. Returns 0 for success, negative error value + * for failure. + */ +static int ti_sci_clk_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct sci_clk_provider *provider; + const struct ti_sci_handle *handle; + int ret; + + handle = devm_ti_sci_get_handle(dev); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL); + if (!provider) + return -ENOMEM; + + provider->sci = handle; + provider->ops = &handle->ops.clk_ops; + provider->dev = dev; + +#ifdef CONFIG_TI_SCI_CLK_PROBE_FROM_FW + ret = ti_sci_scan_clocks_from_fw(provider); + if (ret) { + dev_err(dev, "scan clocks from FW failed: %d\n", ret); + return ret; + } +#else + ret = ti_sci_scan_clocks_from_dt(provider); + if (ret) { + dev_err(dev, "scan clocks from DT failed: %d\n", ret); + return ret; + } +#endif + ret = ti_sci_init_clocks(provider); if (ret) { pr_err("ti-sci-init-clocks failed.\n"); diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index f797f09c6425..ce3d9b300bab 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -300,4 +300,10 @@ config COMMON_CLK_MT8516 help This driver supports MediaTek MT8516 clocks. +config COMMON_CLK_MT8516_AUDSYS + bool "Clock driver for MediaTek MT8516 audsys" + depends on COMMON_CLK_MT8516 + help + This driver supports MediaTek MT8516 audsys clocks. + endmenu diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile index f74937b35f68..672de0099eef 100644 --- a/drivers/clk/mediatek/Makefile +++ b/drivers/clk/mediatek/Makefile @@ -45,3 +45,4 @@ obj-$(CONFIG_COMMON_CLK_MT8183_MMSYS) += clk-mt8183-mm.o obj-$(CONFIG_COMMON_CLK_MT8183_VDECSYS) += clk-mt8183-vdec.o obj-$(CONFIG_COMMON_CLK_MT8183_VENCSYS) += clk-mt8183-venc.o obj-$(CONFIG_COMMON_CLK_MT8516) += clk-mt8516.o +obj-$(CONFIG_COMMON_CLK_MT8516_AUDSYS) += clk-mt8516-aud.o diff --git a/drivers/clk/mediatek/clk-mt8183.c b/drivers/clk/mediatek/clk-mt8183.c index 9d8651033ae9..1aa5f4059251 100644 --- a/drivers/clk/mediatek/clk-mt8183.c +++ b/drivers/clk/mediatek/clk-mt8183.c @@ -395,14 +395,6 @@ static const char * const atb_parents[] = { "syspll_d5" }; -static const char * const sspm_parents[] = { - "clk26m", - "univpll_d2_d4", - "syspll_d2_d2", - "univpll_d2_d2", - "syspll_d3" -}; - static const char * const dpi0_parents[] = { "clk26m", "tvdpll_d2", @@ -606,9 +598,6 @@ static const struct mtk_mux top_muxes[] = { MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_ATB, "atb_sel", atb_parents, 0xa0, 0xa4, 0xa8, 0, 2, 7, 0x004, 24), - MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SSPM, "sspm_sel", - sspm_parents, 0xa0, - 0xa4, 0xa8, 8, 3, 15, 0x004, 25), MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DPI0, "dpi0_sel", dpi0_parents, 0xa0, 0xa4, 0xa8, 16, 4, 23, 0x004, 26), @@ -947,12 +936,8 @@ static const struct mtk_gate infra_clks[] = { "fufs_sel", 13), GATE_INFRA2(CLK_INFRA_MD32_BCLK, "infra_md32_bclk", "axi_sel", 14), - GATE_INFRA2(CLK_INFRA_SSPM, "infra_sspm", - "sspm_sel", 15), GATE_INFRA2(CLK_INFRA_UNIPRO_MBIST, "infra_unipro_mbist", "axi_sel", 16), - GATE_INFRA2(CLK_INFRA_SSPM_BUS_HCLK, "infra_sspm_bus_hclk", - "axi_sel", 17), GATE_INFRA2(CLK_INFRA_I2C5, "infra_i2c5", "i2c_sel", 18), GATE_INFRA2(CLK_INFRA_I2C5_ARBITER, "infra_i2c5_arbiter", @@ -986,10 +971,6 @@ static const struct mtk_gate infra_clks[] = { "msdc50_0_sel", 1), GATE_INFRA3(CLK_INFRA_MSDC2_SELF, "infra_msdc2_self", "msdc50_0_sel", 2), - GATE_INFRA3(CLK_INFRA_SSPM_26M_SELF, "infra_sspm_26m_self", - "f_f26m_ck", 3), - GATE_INFRA3(CLK_INFRA_SSPM_32K_SELF, "infra_sspm_32k_self", - "f_f26m_ck", 4), GATE_INFRA3(CLK_INFRA_UFS_AXI, "infra_ufs_axi", "axi_sel", 5), GATE_INFRA3(CLK_INFRA_I2C6, "infra_i2c6", diff --git a/drivers/clk/mediatek/clk-mt8516-aud.c b/drivers/clk/mediatek/clk-mt8516-aud.c new file mode 100644 index 000000000000..6ab3a06dc9d5 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8516-aud.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 MediaTek Inc. + * Author: James Liao <jamesjj.liao@mediatek.com> + * Fabien Parent <fparent@baylibre.com> + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> + +#include "clk-mtk.h" +#include "clk-gate.h" + +#include <dt-bindings/clock/mt8516-clk.h> + +static const struct mtk_gate_regs aud_cg_regs = { + .set_ofs = 0x0, + .clr_ofs = 0x0, + .sta_ofs = 0x0, +}; + +#define GATE_AUD(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &aud_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_no_setclr, \ + } + +static const struct mtk_gate aud_clks[] __initconst = { + GATE_AUD(CLK_AUD_AFE, "aud_afe", "clk26m_ck", 2), + GATE_AUD(CLK_AUD_I2S, "aud_i2s", "i2s_infra_bck", 6), + GATE_AUD(CLK_AUD_22M, "aud_22m", "rg_aud_engen1", 8), + GATE_AUD(CLK_AUD_24M, "aud_24m", "rg_aud_engen2", 9), + GATE_AUD(CLK_AUD_INTDIR, "aud_intdir", "rg_aud_spdif_in", 15), + GATE_AUD(CLK_AUD_APLL2_TUNER, "aud_apll2_tuner", "rg_aud_engen2", 18), + GATE_AUD(CLK_AUD_APLL_TUNER, "aud_apll_tuner", "rg_aud_engen1", 19), + GATE_AUD(CLK_AUD_HDMI, "aud_hdmi", "apll12_div4", 20), + GATE_AUD(CLK_AUD_SPDF, "aud_spdf", "apll12_div6", 21), + GATE_AUD(CLK_AUD_ADC, "aud_adc", "aud_afe", 24), + GATE_AUD(CLK_AUD_DAC, "aud_dac", "aud_afe", 25), + GATE_AUD(CLK_AUD_DAC_PREDIS, "aud_dac_predis", "aud_afe", 26), + GATE_AUD(CLK_AUD_TML, "aud_tml", "aud_afe", 27), +}; + +static void __init mtk_audsys_init(struct device_node *node) +{ + struct clk_onecell_data *clk_data; + int r; + + clk_data = mtk_alloc_clk_data(CLK_AUD_NR_CLK); + + mtk_clk_register_gates(node, aud_clks, ARRAY_SIZE(aud_clks), clk_data); + + r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data); + if (r) + pr_err("%s(): could not register clock provider: %d\n", + __func__, r); + +} +CLK_OF_DECLARE(mtk_audsys, "mediatek,mt8516-audsys", mtk_audsys_init); diff --git a/drivers/clk/mediatek/clk-mt8516.c b/drivers/clk/mediatek/clk-mt8516.c index 26fe43cc9ea2..9d4261ecc760 100644 --- a/drivers/clk/mediatek/clk-mt8516.c +++ b/drivers/clk/mediatek/clk-mt8516.c @@ -231,11 +231,6 @@ static const char * const nfi1x_pad_parents[] __initconst = { "nfi1x_ck" }; -static const char * const ddrphycfg_parents[] __initconst = { - "clk26m_ck", - "mainpll_d16" -}; - static const char * const usb_78m_parents[] __initconst = { "clk_null", "clk26m_ck", diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c index 7a8ef80e5f2c..3ddd0efc9ee0 100644 --- a/drivers/clk/meson/axg.c +++ b/drivers/clk/meson/axg.c @@ -469,11 +469,6 @@ static struct clk_regmap axg_mpll0_div = { .shift = 16, .width = 9, }, - .ssen = { - .reg_off = HHI_MPLL_CNTL, - .shift = 25, - .width = 1, - }, .misc = { .reg_off = HHI_PLL_TOP_MISC, .shift = 0, @@ -568,6 +563,11 @@ static struct clk_regmap axg_mpll2_div = { .shift = 16, .width = 9, }, + .ssen = { + .reg_off = HHI_MPLL_CNTL, + .shift = 25, + .width = 1, + }, .misc = { .reg_off = HHI_PLL_TOP_MISC, .shift = 2, diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c index f76850d99e59..2d39a8bc367c 100644 --- a/drivers/clk/meson/clk-mpll.c +++ b/drivers/clk/meson/clk-mpll.c @@ -115,21 +115,12 @@ static int mpll_set_rate(struct clk_hw *hw, else __acquire(mpll->lock); - /* Enable and set the fractional part */ + /* Set the fractional part */ meson_parm_write(clk->map, &mpll->sdm, sdm); - meson_parm_write(clk->map, &mpll->sdm_en, 1); - - /* Set additional fractional part enable if required */ - if (MESON_PARM_APPLICABLE(&mpll->ssen)) - meson_parm_write(clk->map, &mpll->ssen, 1); /* Set the integer divider part */ meson_parm_write(clk->map, &mpll->n2, n2); - /* Set the magic misc bit if required */ - if (MESON_PARM_APPLICABLE(&mpll->misc)) - meson_parm_write(clk->map, &mpll->misc, 1); - if (mpll->lock) spin_unlock_irqrestore(mpll->lock, flags); else @@ -138,6 +129,30 @@ static int mpll_set_rate(struct clk_hw *hw, return 0; } +static void mpll_init(struct clk_hw *hw) +{ + struct clk_regmap *clk = to_clk_regmap(hw); + struct meson_clk_mpll_data *mpll = meson_clk_mpll_data(clk); + + if (mpll->init_count) + regmap_multi_reg_write(clk->map, mpll->init_regs, + mpll->init_count); + + /* Enable the fractional part */ + meson_parm_write(clk->map, &mpll->sdm_en, 1); + + /* Set spread spectrum if possible */ + if (MESON_PARM_APPLICABLE(&mpll->ssen)) { + unsigned int ss = + mpll->flags & CLK_MESON_MPLL_SPREAD_SPECTRUM ? 1 : 0; + meson_parm_write(clk->map, &mpll->ssen, ss); + } + + /* Set the magic misc bit if required */ + if (MESON_PARM_APPLICABLE(&mpll->misc)) + meson_parm_write(clk->map, &mpll->misc, 1); +} + const struct clk_ops meson_clk_mpll_ro_ops = { .recalc_rate = mpll_recalc_rate, .round_rate = mpll_round_rate, @@ -148,6 +163,7 @@ const struct clk_ops meson_clk_mpll_ops = { .recalc_rate = mpll_recalc_rate, .round_rate = mpll_round_rate, .set_rate = mpll_set_rate, + .init = mpll_init, }; EXPORT_SYMBOL_GPL(meson_clk_mpll_ops); diff --git a/drivers/clk/meson/clk-mpll.h b/drivers/clk/meson/clk-mpll.h index cf79340006dd..a991d568c43a 100644 --- a/drivers/clk/meson/clk-mpll.h +++ b/drivers/clk/meson/clk-mpll.h @@ -18,11 +18,14 @@ struct meson_clk_mpll_data { struct parm n2; struct parm ssen; struct parm misc; + const struct reg_sequence *init_regs; + unsigned int init_count; spinlock_t *lock; u8 flags; }; #define CLK_MESON_MPLL_ROUND_CLOSEST BIT(0) +#define CLK_MESON_MPLL_SPREAD_SPECTRUM BIT(1) extern const struct clk_ops meson_clk_mpll_ro_ops; extern const struct clk_ops meson_clk_mpll_ops; diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 206fafd299ea..db1c4ed9d54e 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -150,6 +150,57 @@ static struct clk_regmap g12a_sys_pll = { }, }; +static struct clk_regmap g12b_sys1_pll_dco = { + .data = &(struct meson_clk_pll_data){ + .en = { + .reg_off = HHI_SYS1_PLL_CNTL0, + .shift = 28, + .width = 1, + }, + .m = { + .reg_off = HHI_SYS1_PLL_CNTL0, + .shift = 0, + .width = 8, + }, + .n = { + .reg_off = HHI_SYS1_PLL_CNTL0, + .shift = 10, + .width = 5, + }, + .l = { + .reg_off = HHI_SYS1_PLL_CNTL0, + .shift = 31, + .width = 1, + }, + .rst = { + .reg_off = HHI_SYS1_PLL_CNTL0, + .shift = 29, + .width = 1, + }, + }, + .hw.init = &(struct clk_init_data){ + .name = "sys1_pll_dco", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12b_sys1_pll = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_SYS1_PLL_CNTL0, + .shift = 16, + .width = 3, + .flags = CLK_DIVIDER_POWER_OF_TWO, + }, + .hw.init = &(struct clk_init_data){ + .name = "sys1_pll", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "sys1_pll_dco" }, + .num_parents = 1, + }, +}; + static struct clk_regmap g12a_sys_pll_div16_en = { .data = &(struct clk_regmap_gate_data){ .offset = HHI_SYS_CPU_CLK_CNTL1, @@ -167,6 +218,23 @@ static struct clk_regmap g12a_sys_pll_div16_en = { }, }; +static struct clk_regmap g12b_sys1_pll_div16_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .bit_idx = 24, + }, + .hw.init = &(struct clk_init_data) { + .name = "sys1_pll_div16_en", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ "sys1_pll" }, + .num_parents = 1, + /* + * This clock is used to debug the sys_pll range + * Linux should not change it at runtime + */ + }, +}; + static struct clk_fixed_factor g12a_sys_pll_div16 = { .mult = 1, .div = 16, @@ -178,6 +246,17 @@ static struct clk_fixed_factor g12a_sys_pll_div16 = { }, }; +static struct clk_fixed_factor g12b_sys1_pll_div16 = { + .mult = 1, + .div = 16, + .hw.init = &(struct clk_init_data){ + .name = "sys1_pll_div16", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "sys1_pll_div16_en" }, + .num_parents = 1, + }, +}; + /* Datasheet names this field as "premux0" */ static struct clk_regmap g12a_cpu_clk_premux0 = { .data = &(struct clk_regmap_mux_data){ @@ -306,6 +385,150 @@ static struct clk_regmap g12a_cpu_clk = { }, }; +/* Datasheet names this field as "Final_mux_sel" */ +static struct clk_regmap g12b_cpu_clk = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPU_CLK_CNTL0, + .mask = 0x1, + .shift = 11, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpu_clk", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpu_clk_dyn", + "sys1_pll" }, + .num_parents = 2, + }, +}; + +/* Datasheet names this field as "premux0" */ +static struct clk_regmap g12b_cpub_clk_premux0 = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .mask = 0x3, + .shift = 0, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_dyn0_sel", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal", + "fclk_div2", + "fclk_div3" }, + .num_parents = 3, + }, +}; + +/* Datasheet names this field as "mux0_divn_tcnt" */ +static struct clk_regmap g12b_cpub_clk_mux0_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .shift = 4, + .width = 6, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_dyn0_div", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_dyn0_sel" }, + .num_parents = 1, + }, +}; + +/* Datasheet names this field as "postmux0" */ +static struct clk_regmap g12b_cpub_clk_postmux0 = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .mask = 0x1, + .shift = 2, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_dyn0", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_dyn0_sel", + "cpub_clk_dyn0_div" }, + .num_parents = 2, + }, +}; + +/* Datasheet names this field as "premux1" */ +static struct clk_regmap g12b_cpub_clk_premux1 = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .mask = 0x3, + .shift = 16, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_dyn1_sel", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ IN_PREFIX "xtal", + "fclk_div2", + "fclk_div3" }, + .num_parents = 3, + }, +}; + +/* Datasheet names this field as "Mux1_divn_tcnt" */ +static struct clk_regmap g12b_cpub_clk_mux1_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .shift = 20, + .width = 6, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_dyn1_div", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_dyn1_sel" }, + .num_parents = 1, + }, +}; + +/* Datasheet names this field as "postmux1" */ +static struct clk_regmap g12b_cpub_clk_postmux1 = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .mask = 0x1, + .shift = 18, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_dyn1", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_dyn1_sel", + "cpub_clk_dyn1_div" }, + .num_parents = 2, + }, +}; + +/* Datasheet names this field as "Final_dyn_mux_sel" */ +static struct clk_regmap g12b_cpub_clk_dyn = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .mask = 0x1, + .shift = 10, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_dyn", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_dyn0", + "cpub_clk_dyn1" }, + .num_parents = 2, + }, +}; + +/* Datasheet names this field as "Final_mux_sel" */ +static struct clk_regmap g12b_cpub_clk = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL, + .mask = 0x1, + .shift = 11, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_dyn", + "sys_pll" }, + .num_parents = 2, + }, +}; + static struct clk_regmap g12a_cpu_clk_div16_en = { .data = &(struct clk_regmap_gate_data){ .offset = HHI_SYS_CPU_CLK_CNTL1, @@ -323,6 +546,23 @@ static struct clk_regmap g12a_cpu_clk_div16_en = { }, }; +static struct clk_regmap g12b_cpub_clk_div16_en = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .bit_idx = 1, + }, + .hw.init = &(struct clk_init_data) { + .name = "cpub_clk_div16_en", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + /* + * This clock is used to debug the cpu_clk range + * Linux should not change it at runtime + */ + }, +}; + static struct clk_fixed_factor g12a_cpu_clk_div16 = { .mult = 1, .div = 16, @@ -334,6 +574,17 @@ static struct clk_fixed_factor g12a_cpu_clk_div16 = { }, }; +static struct clk_fixed_factor g12b_cpub_clk_div16 = { + .mult = 1, + .div = 16, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div16", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk_div16_en" }, + .num_parents = 1, + }, +}; + static struct clk_regmap g12a_cpu_clk_apb_div = { .data = &(struct clk_regmap_div_data){ .offset = HHI_SYS_CPU_CLK_CNTL1, @@ -462,6 +713,240 @@ static struct clk_regmap g12a_cpu_clk_trace = { }, }; +static struct clk_fixed_factor g12b_cpub_clk_div2 = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div2", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12b_cpub_clk_div3 = { + .mult = 1, + .div = 3, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div3", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12b_cpub_clk_div4 = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div4", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12b_cpub_clk_div5 = { + .mult = 1, + .div = 5, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div5", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12b_cpub_clk_div6 = { + .mult = 1, + .div = 6, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div6", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12b_cpub_clk_div7 = { + .mult = 1, + .div = 7, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div7", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor g12b_cpub_clk_div8 = { + .mult = 1, + .div = 8, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_div8", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "cpub_clk" }, + .num_parents = 1, + }, +}; + +static u32 mux_table_cpub[] = { 1, 2, 3, 4, 5, 6, 7 }; +static struct clk_regmap g12b_cpub_clk_apb_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .mask = 7, + .shift = 3, + .table = mux_table_cpub, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_apb_sel", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_div2", + "cpub_clk_div3", + "cpub_clk_div4", + "cpub_clk_div5", + "cpub_clk_div6", + "cpub_clk_div7", + "cpub_clk_div8" }, + .num_parents = 7, + }, +}; + +static struct clk_regmap g12b_cpub_clk_apb = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .bit_idx = 16, + .flags = CLK_GATE_SET_TO_DISABLE, + }, + .hw.init = &(struct clk_init_data) { + .name = "cpub_clk_apb", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_apb_sel" }, + .num_parents = 1, + /* + * This clock is set by the ROM monitor code, + * Linux should not change it at runtime + */ + }, +}; + +static struct clk_regmap g12b_cpub_clk_atb_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .mask = 7, + .shift = 6, + .table = mux_table_cpub, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_atb_sel", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_div2", + "cpub_clk_div3", + "cpub_clk_div4", + "cpub_clk_div5", + "cpub_clk_div6", + "cpub_clk_div7", + "cpub_clk_div8" }, + .num_parents = 7, + }, +}; + +static struct clk_regmap g12b_cpub_clk_atb = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .bit_idx = 17, + .flags = CLK_GATE_SET_TO_DISABLE, + }, + .hw.init = &(struct clk_init_data) { + .name = "cpub_clk_atb", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_atb_sel" }, + .num_parents = 1, + /* + * This clock is set by the ROM monitor code, + * Linux should not change it at runtime + */ + }, +}; + +static struct clk_regmap g12b_cpub_clk_axi_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .mask = 7, + .shift = 9, + .table = mux_table_cpub, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_axi_sel", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_div2", + "cpub_clk_div3", + "cpub_clk_div4", + "cpub_clk_div5", + "cpub_clk_div6", + "cpub_clk_div7", + "cpub_clk_div8" }, + .num_parents = 7, + }, +}; + +static struct clk_regmap g12b_cpub_clk_axi = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .bit_idx = 18, + .flags = CLK_GATE_SET_TO_DISABLE, + }, + .hw.init = &(struct clk_init_data) { + .name = "cpub_clk_axi", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_axi_sel" }, + .num_parents = 1, + /* + * This clock is set by the ROM monitor code, + * Linux should not change it at runtime + */ + }, +}; + +static struct clk_regmap g12b_cpub_clk_trace_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .mask = 7, + .shift = 20, + .table = mux_table_cpub, + }, + .hw.init = &(struct clk_init_data){ + .name = "cpub_clk_trace_sel", + .ops = &clk_regmap_mux_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_div2", + "cpub_clk_div3", + "cpub_clk_div4", + "cpub_clk_div5", + "cpub_clk_div6", + "cpub_clk_div7", + "cpub_clk_div8" }, + .num_parents = 7, + }, +}; + +static struct clk_regmap g12b_cpub_clk_trace = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_SYS_CPUB_CLK_CNTL1, + .bit_idx = 23, + .flags = CLK_GATE_SET_TO_DISABLE, + }, + .hw.init = &(struct clk_init_data) { + .name = "cpub_clk_trace", + .ops = &clk_regmap_gate_ro_ops, + .parent_names = (const char *[]){ "cpub_clk_trace_sel" }, + .num_parents = 1, + /* + * This clock is set by the ROM monitor code, + * Linux should not change it at runtime + */ + }, +}; + static const struct pll_mult_range g12a_gp0_pll_mult_range = { .min = 55, .max = 255, @@ -865,6 +1350,16 @@ static struct clk_regmap g12a_fclk_div3 = { .ops = &clk_regmap_gate_ops, .parent_names = (const char *[]){ "fclk_div3_div" }, .num_parents = 1, + /* + * This clock is used by the resident firmware and is required + * by the platform to operate correctly. + * Until the following condition are met, we need this clock to + * be marked as critical: + * a) Mark the clock used by a firmware resource, if possible + * b) CCF has a clock hand-off mechanism to make the sure the + * clock stays on until the proper driver comes along + */ + .flags = CLK_IS_CRITICAL, }, }; @@ -1001,6 +1496,10 @@ static struct clk_fixed_factor g12a_mpll_prediv = { }, }; +static const struct reg_sequence g12a_mpll0_init_regs[] = { + { .reg = HHI_MPLL_CNTL2, .def = 0x40000033 }, +}; + static struct clk_regmap g12a_mpll0_div = { .data = &(struct meson_clk_mpll_data){ .sdm = { @@ -1024,6 +1523,8 @@ static struct clk_regmap g12a_mpll0_div = { .width = 1, }, .lock = &meson_clk_lock, + .init_regs = g12a_mpll0_init_regs, + .init_count = ARRAY_SIZE(g12a_mpll0_init_regs), }, .hw.init = &(struct clk_init_data){ .name = "mpll0_div", @@ -1047,6 +1548,10 @@ static struct clk_regmap g12a_mpll0 = { }, }; +static const struct reg_sequence g12a_mpll1_init_regs[] = { + { .reg = HHI_MPLL_CNTL4, .def = 0x40000033 }, +}; + static struct clk_regmap g12a_mpll1_div = { .data = &(struct meson_clk_mpll_data){ .sdm = { @@ -1070,6 +1575,8 @@ static struct clk_regmap g12a_mpll1_div = { .width = 1, }, .lock = &meson_clk_lock, + .init_regs = g12a_mpll1_init_regs, + .init_count = ARRAY_SIZE(g12a_mpll1_init_regs), }, .hw.init = &(struct clk_init_data){ .name = "mpll1_div", @@ -1093,6 +1600,10 @@ static struct clk_regmap g12a_mpll1 = { }, }; +static const struct reg_sequence g12a_mpll2_init_regs[] = { + { .reg = HHI_MPLL_CNTL6, .def = 0x40000033 }, +}; + static struct clk_regmap g12a_mpll2_div = { .data = &(struct meson_clk_mpll_data){ .sdm = { @@ -1116,6 +1627,8 @@ static struct clk_regmap g12a_mpll2_div = { .width = 1, }, .lock = &meson_clk_lock, + .init_regs = g12a_mpll2_init_regs, + .init_count = ARRAY_SIZE(g12a_mpll2_init_regs), }, .hw.init = &(struct clk_init_data){ .name = "mpll2_div", @@ -1139,6 +1652,10 @@ static struct clk_regmap g12a_mpll2 = { }, }; +static const struct reg_sequence g12a_mpll3_init_regs[] = { + { .reg = HHI_MPLL_CNTL8, .def = 0x40000033 }, +}; + static struct clk_regmap g12a_mpll3_div = { .data = &(struct meson_clk_mpll_data){ .sdm = { @@ -1162,6 +1679,8 @@ static struct clk_regmap g12a_mpll3_div = { .width = 1, }, .lock = &meson_clk_lock, + .init_regs = g12a_mpll3_init_regs, + .init_count = ARRAY_SIZE(g12a_mpll3_init_regs), }, .hw.init = &(struct clk_init_data){ .name = "mpll3_div", @@ -2480,6 +2999,33 @@ static struct clk_regmap g12a_mali = { }, }; +static struct clk_regmap g12a_ts_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_TS_CLK_CNTL, + .shift = 0, + .width = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "ts_div", + .ops = &clk_regmap_divider_ro_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + }, +}; + +static struct clk_regmap g12a_ts = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_TS_CLK_CNTL, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "ts", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "ts_div" }, + .num_parents = 1, + }, +}; + /* Everything Else (EE) domain gates */ static MESON_GATE(g12a_ddr, HHI_GCLK_MPEG0, 0); static MESON_GATE(g12a_dos, HHI_GCLK_MPEG0, 1); @@ -2769,6 +3315,257 @@ static struct clk_hw_onecell_data g12a_hw_onecell_data = { [CLKID_VDEC_HEVCF_SEL] = &g12a_vdec_hevcf_sel.hw, [CLKID_VDEC_HEVCF_DIV] = &g12a_vdec_hevcf_div.hw, [CLKID_VDEC_HEVCF] = &g12a_vdec_hevcf.hw, + [CLKID_TS_DIV] = &g12a_ts_div.hw, + [CLKID_TS] = &g12a_ts.hw, + [NR_CLKS] = NULL, + }, + .num = NR_CLKS, +}; + +static struct clk_hw_onecell_data g12b_hw_onecell_data = { + .hws = { + [CLKID_SYS_PLL] = &g12a_sys_pll.hw, + [CLKID_FIXED_PLL] = &g12a_fixed_pll.hw, + [CLKID_FCLK_DIV2] = &g12a_fclk_div2.hw, + [CLKID_FCLK_DIV3] = &g12a_fclk_div3.hw, + [CLKID_FCLK_DIV4] = &g12a_fclk_div4.hw, + [CLKID_FCLK_DIV5] = &g12a_fclk_div5.hw, + [CLKID_FCLK_DIV7] = &g12a_fclk_div7.hw, + [CLKID_FCLK_DIV2P5] = &g12a_fclk_div2p5.hw, + [CLKID_GP0_PLL] = &g12a_gp0_pll.hw, + [CLKID_MPEG_SEL] = &g12a_mpeg_clk_sel.hw, + [CLKID_MPEG_DIV] = &g12a_mpeg_clk_div.hw, + [CLKID_CLK81] = &g12a_clk81.hw, + [CLKID_MPLL0] = &g12a_mpll0.hw, + [CLKID_MPLL1] = &g12a_mpll1.hw, + [CLKID_MPLL2] = &g12a_mpll2.hw, + [CLKID_MPLL3] = &g12a_mpll3.hw, + [CLKID_DDR] = &g12a_ddr.hw, + [CLKID_DOS] = &g12a_dos.hw, + [CLKID_AUDIO_LOCKER] = &g12a_audio_locker.hw, + [CLKID_MIPI_DSI_HOST] = &g12a_mipi_dsi_host.hw, + [CLKID_ETH_PHY] = &g12a_eth_phy.hw, + [CLKID_ISA] = &g12a_isa.hw, + [CLKID_PL301] = &g12a_pl301.hw, + [CLKID_PERIPHS] = &g12a_periphs.hw, + [CLKID_SPICC0] = &g12a_spicc_0.hw, + [CLKID_I2C] = &g12a_i2c.hw, + [CLKID_SANA] = &g12a_sana.hw, + [CLKID_SD] = &g12a_sd.hw, + [CLKID_RNG0] = &g12a_rng0.hw, + [CLKID_UART0] = &g12a_uart0.hw, + [CLKID_SPICC1] = &g12a_spicc_1.hw, + [CLKID_HIU_IFACE] = &g12a_hiu_reg.hw, + [CLKID_MIPI_DSI_PHY] = &g12a_mipi_dsi_phy.hw, + [CLKID_ASSIST_MISC] = &g12a_assist_misc.hw, + [CLKID_SD_EMMC_A] = &g12a_emmc_a.hw, + [CLKID_SD_EMMC_B] = &g12a_emmc_b.hw, + [CLKID_SD_EMMC_C] = &g12a_emmc_c.hw, + [CLKID_AUDIO_CODEC] = &g12a_audio_codec.hw, + [CLKID_AUDIO] = &g12a_audio.hw, + [CLKID_ETH] = &g12a_eth_core.hw, + [CLKID_DEMUX] = &g12a_demux.hw, + [CLKID_AUDIO_IFIFO] = &g12a_audio_ififo.hw, + [CLKID_ADC] = &g12a_adc.hw, + [CLKID_UART1] = &g12a_uart1.hw, + [CLKID_G2D] = &g12a_g2d.hw, + [CLKID_RESET] = &g12a_reset.hw, + [CLKID_PCIE_COMB] = &g12a_pcie_comb.hw, + [CLKID_PARSER] = &g12a_parser.hw, + [CLKID_USB] = &g12a_usb_general.hw, + [CLKID_PCIE_PHY] = &g12a_pcie_phy.hw, + [CLKID_AHB_ARB0] = &g12a_ahb_arb0.hw, + [CLKID_AHB_DATA_BUS] = &g12a_ahb_data_bus.hw, + [CLKID_AHB_CTRL_BUS] = &g12a_ahb_ctrl_bus.hw, + [CLKID_HTX_HDCP22] = &g12a_htx_hdcp22.hw, + [CLKID_HTX_PCLK] = &g12a_htx_pclk.hw, + [CLKID_BT656] = &g12a_bt656.hw, + [CLKID_USB1_DDR_BRIDGE] = &g12a_usb1_to_ddr.hw, + [CLKID_MMC_PCLK] = &g12a_mmc_pclk.hw, + [CLKID_UART2] = &g12a_uart2.hw, + [CLKID_VPU_INTR] = &g12a_vpu_intr.hw, + [CLKID_GIC] = &g12a_gic.hw, + [CLKID_SD_EMMC_A_CLK0_SEL] = &g12a_sd_emmc_a_clk0_sel.hw, + [CLKID_SD_EMMC_A_CLK0_DIV] = &g12a_sd_emmc_a_clk0_div.hw, + [CLKID_SD_EMMC_A_CLK0] = &g12a_sd_emmc_a_clk0.hw, + [CLKID_SD_EMMC_B_CLK0_SEL] = &g12a_sd_emmc_b_clk0_sel.hw, + [CLKID_SD_EMMC_B_CLK0_DIV] = &g12a_sd_emmc_b_clk0_div.hw, + [CLKID_SD_EMMC_B_CLK0] = &g12a_sd_emmc_b_clk0.hw, + [CLKID_SD_EMMC_C_CLK0_SEL] = &g12a_sd_emmc_c_clk0_sel.hw, + [CLKID_SD_EMMC_C_CLK0_DIV] = &g12a_sd_emmc_c_clk0_div.hw, + [CLKID_SD_EMMC_C_CLK0] = &g12a_sd_emmc_c_clk0.hw, + [CLKID_MPLL0_DIV] = &g12a_mpll0_div.hw, + [CLKID_MPLL1_DIV] = &g12a_mpll1_div.hw, + [CLKID_MPLL2_DIV] = &g12a_mpll2_div.hw, + [CLKID_MPLL3_DIV] = &g12a_mpll3_div.hw, + [CLKID_FCLK_DIV2_DIV] = &g12a_fclk_div2_div.hw, + [CLKID_FCLK_DIV3_DIV] = &g12a_fclk_div3_div.hw, + [CLKID_FCLK_DIV4_DIV] = &g12a_fclk_div4_div.hw, + [CLKID_FCLK_DIV5_DIV] = &g12a_fclk_div5_div.hw, + [CLKID_FCLK_DIV7_DIV] = &g12a_fclk_div7_div.hw, + [CLKID_FCLK_DIV2P5_DIV] = &g12a_fclk_div2p5_div.hw, + [CLKID_HIFI_PLL] = &g12a_hifi_pll.hw, + [CLKID_VCLK2_VENCI0] = &g12a_vclk2_venci0.hw, + [CLKID_VCLK2_VENCI1] = &g12a_vclk2_venci1.hw, + [CLKID_VCLK2_VENCP0] = &g12a_vclk2_vencp0.hw, + [CLKID_VCLK2_VENCP1] = &g12a_vclk2_vencp1.hw, + [CLKID_VCLK2_VENCT0] = &g12a_vclk2_venct0.hw, + [CLKID_VCLK2_VENCT1] = &g12a_vclk2_venct1.hw, + [CLKID_VCLK2_OTHER] = &g12a_vclk2_other.hw, + [CLKID_VCLK2_ENCI] = &g12a_vclk2_enci.hw, + [CLKID_VCLK2_ENCP] = &g12a_vclk2_encp.hw, + [CLKID_DAC_CLK] = &g12a_dac_clk.hw, + [CLKID_AOCLK] = &g12a_aoclk_gate.hw, + [CLKID_IEC958] = &g12a_iec958_gate.hw, + [CLKID_ENC480P] = &g12a_enc480p.hw, + [CLKID_RNG1] = &g12a_rng1.hw, + [CLKID_VCLK2_ENCT] = &g12a_vclk2_enct.hw, + [CLKID_VCLK2_ENCL] = &g12a_vclk2_encl.hw, + [CLKID_VCLK2_VENCLMMC] = &g12a_vclk2_venclmmc.hw, + [CLKID_VCLK2_VENCL] = &g12a_vclk2_vencl.hw, + [CLKID_VCLK2_OTHER1] = &g12a_vclk2_other1.hw, + [CLKID_FIXED_PLL_DCO] = &g12a_fixed_pll_dco.hw, + [CLKID_SYS_PLL_DCO] = &g12a_sys_pll_dco.hw, + [CLKID_GP0_PLL_DCO] = &g12a_gp0_pll_dco.hw, + [CLKID_HIFI_PLL_DCO] = &g12a_hifi_pll_dco.hw, + [CLKID_DMA] = &g12a_dma.hw, + [CLKID_EFUSE] = &g12a_efuse.hw, + [CLKID_ROM_BOOT] = &g12a_rom_boot.hw, + [CLKID_RESET_SEC] = &g12a_reset_sec.hw, + [CLKID_SEC_AHB_APB3] = &g12a_sec_ahb_apb3.hw, + [CLKID_MPLL_PREDIV] = &g12a_mpll_prediv.hw, + [CLKID_VPU_0_SEL] = &g12a_vpu_0_sel.hw, + [CLKID_VPU_0_DIV] = &g12a_vpu_0_div.hw, + [CLKID_VPU_0] = &g12a_vpu_0.hw, + [CLKID_VPU_1_SEL] = &g12a_vpu_1_sel.hw, + [CLKID_VPU_1_DIV] = &g12a_vpu_1_div.hw, + [CLKID_VPU_1] = &g12a_vpu_1.hw, + [CLKID_VPU] = &g12a_vpu.hw, + [CLKID_VAPB_0_SEL] = &g12a_vapb_0_sel.hw, + [CLKID_VAPB_0_DIV] = &g12a_vapb_0_div.hw, + [CLKID_VAPB_0] = &g12a_vapb_0.hw, + [CLKID_VAPB_1_SEL] = &g12a_vapb_1_sel.hw, + [CLKID_VAPB_1_DIV] = &g12a_vapb_1_div.hw, + [CLKID_VAPB_1] = &g12a_vapb_1.hw, + [CLKID_VAPB_SEL] = &g12a_vapb_sel.hw, + [CLKID_VAPB] = &g12a_vapb.hw, + [CLKID_HDMI_PLL_DCO] = &g12a_hdmi_pll_dco.hw, + [CLKID_HDMI_PLL_OD] = &g12a_hdmi_pll_od.hw, + [CLKID_HDMI_PLL_OD2] = &g12a_hdmi_pll_od2.hw, + [CLKID_HDMI_PLL] = &g12a_hdmi_pll.hw, + [CLKID_VID_PLL] = &g12a_vid_pll_div.hw, + [CLKID_VID_PLL_SEL] = &g12a_vid_pll_sel.hw, + [CLKID_VID_PLL_DIV] = &g12a_vid_pll.hw, + [CLKID_VCLK_SEL] = &g12a_vclk_sel.hw, + [CLKID_VCLK2_SEL] = &g12a_vclk2_sel.hw, + [CLKID_VCLK_INPUT] = &g12a_vclk_input.hw, + [CLKID_VCLK2_INPUT] = &g12a_vclk2_input.hw, + [CLKID_VCLK_DIV] = &g12a_vclk_div.hw, + [CLKID_VCLK2_DIV] = &g12a_vclk2_div.hw, + [CLKID_VCLK] = &g12a_vclk.hw, + [CLKID_VCLK2] = &g12a_vclk2.hw, + [CLKID_VCLK_DIV1] = &g12a_vclk_div1.hw, + [CLKID_VCLK_DIV2_EN] = &g12a_vclk_div2_en.hw, + [CLKID_VCLK_DIV4_EN] = &g12a_vclk_div4_en.hw, + [CLKID_VCLK_DIV6_EN] = &g12a_vclk_div6_en.hw, + [CLKID_VCLK_DIV12_EN] = &g12a_vclk_div12_en.hw, + [CLKID_VCLK2_DIV1] = &g12a_vclk2_div1.hw, + [CLKID_VCLK2_DIV2_EN] = &g12a_vclk2_div2_en.hw, + [CLKID_VCLK2_DIV4_EN] = &g12a_vclk2_div4_en.hw, + [CLKID_VCLK2_DIV6_EN] = &g12a_vclk2_div6_en.hw, + [CLKID_VCLK2_DIV12_EN] = &g12a_vclk2_div12_en.hw, + [CLKID_VCLK_DIV2] = &g12a_vclk_div2.hw, + [CLKID_VCLK_DIV4] = &g12a_vclk_div4.hw, + [CLKID_VCLK_DIV6] = &g12a_vclk_div6.hw, + [CLKID_VCLK_DIV12] = &g12a_vclk_div12.hw, + [CLKID_VCLK2_DIV2] = &g12a_vclk2_div2.hw, + [CLKID_VCLK2_DIV4] = &g12a_vclk2_div4.hw, + [CLKID_VCLK2_DIV6] = &g12a_vclk2_div6.hw, + [CLKID_VCLK2_DIV12] = &g12a_vclk2_div12.hw, + [CLKID_CTS_ENCI_SEL] = &g12a_cts_enci_sel.hw, + [CLKID_CTS_ENCP_SEL] = &g12a_cts_encp_sel.hw, + [CLKID_CTS_VDAC_SEL] = &g12a_cts_vdac_sel.hw, + [CLKID_HDMI_TX_SEL] = &g12a_hdmi_tx_sel.hw, + [CLKID_CTS_ENCI] = &g12a_cts_enci.hw, + [CLKID_CTS_ENCP] = &g12a_cts_encp.hw, + [CLKID_CTS_VDAC] = &g12a_cts_vdac.hw, + [CLKID_HDMI_TX] = &g12a_hdmi_tx.hw, + [CLKID_HDMI_SEL] = &g12a_hdmi_sel.hw, + [CLKID_HDMI_DIV] = &g12a_hdmi_div.hw, + [CLKID_HDMI] = &g12a_hdmi.hw, + [CLKID_MALI_0_SEL] = &g12a_mali_0_sel.hw, + [CLKID_MALI_0_DIV] = &g12a_mali_0_div.hw, + [CLKID_MALI_0] = &g12a_mali_0.hw, + [CLKID_MALI_1_SEL] = &g12a_mali_1_sel.hw, + [CLKID_MALI_1_DIV] = &g12a_mali_1_div.hw, + [CLKID_MALI_1] = &g12a_mali_1.hw, + [CLKID_MALI] = &g12a_mali.hw, + [CLKID_MPLL_50M_DIV] = &g12a_mpll_50m_div.hw, + [CLKID_MPLL_50M] = &g12a_mpll_50m.hw, + [CLKID_SYS_PLL_DIV16_EN] = &g12a_sys_pll_div16_en.hw, + [CLKID_SYS_PLL_DIV16] = &g12a_sys_pll_div16.hw, + [CLKID_CPU_CLK_DYN0_SEL] = &g12a_cpu_clk_premux0.hw, + [CLKID_CPU_CLK_DYN0_DIV] = &g12a_cpu_clk_mux0_div.hw, + [CLKID_CPU_CLK_DYN0] = &g12a_cpu_clk_postmux0.hw, + [CLKID_CPU_CLK_DYN1_SEL] = &g12a_cpu_clk_premux1.hw, + [CLKID_CPU_CLK_DYN1_DIV] = &g12a_cpu_clk_mux1_div.hw, + [CLKID_CPU_CLK_DYN1] = &g12a_cpu_clk_postmux1.hw, + [CLKID_CPU_CLK_DYN] = &g12a_cpu_clk_dyn.hw, + [CLKID_CPU_CLK] = &g12b_cpu_clk.hw, + [CLKID_CPU_CLK_DIV16_EN] = &g12a_cpu_clk_div16_en.hw, + [CLKID_CPU_CLK_DIV16] = &g12a_cpu_clk_div16.hw, + [CLKID_CPU_CLK_APB_DIV] = &g12a_cpu_clk_apb_div.hw, + [CLKID_CPU_CLK_APB] = &g12a_cpu_clk_apb.hw, + [CLKID_CPU_CLK_ATB_DIV] = &g12a_cpu_clk_atb_div.hw, + [CLKID_CPU_CLK_ATB] = &g12a_cpu_clk_atb.hw, + [CLKID_CPU_CLK_AXI_DIV] = &g12a_cpu_clk_axi_div.hw, + [CLKID_CPU_CLK_AXI] = &g12a_cpu_clk_axi.hw, + [CLKID_CPU_CLK_TRACE_DIV] = &g12a_cpu_clk_trace_div.hw, + [CLKID_CPU_CLK_TRACE] = &g12a_cpu_clk_trace.hw, + [CLKID_PCIE_PLL_DCO] = &g12a_pcie_pll_dco.hw, + [CLKID_PCIE_PLL_DCO_DIV2] = &g12a_pcie_pll_dco_div2.hw, + [CLKID_PCIE_PLL_OD] = &g12a_pcie_pll_od.hw, + [CLKID_PCIE_PLL] = &g12a_pcie_pll.hw, + [CLKID_VDEC_1_SEL] = &g12a_vdec_1_sel.hw, + [CLKID_VDEC_1_DIV] = &g12a_vdec_1_div.hw, + [CLKID_VDEC_1] = &g12a_vdec_1.hw, + [CLKID_VDEC_HEVC_SEL] = &g12a_vdec_hevc_sel.hw, + [CLKID_VDEC_HEVC_DIV] = &g12a_vdec_hevc_div.hw, + [CLKID_VDEC_HEVC] = &g12a_vdec_hevc.hw, + [CLKID_VDEC_HEVCF_SEL] = &g12a_vdec_hevcf_sel.hw, + [CLKID_VDEC_HEVCF_DIV] = &g12a_vdec_hevcf_div.hw, + [CLKID_VDEC_HEVCF] = &g12a_vdec_hevcf.hw, + [CLKID_TS_DIV] = &g12a_ts_div.hw, + [CLKID_TS] = &g12a_ts.hw, + [CLKID_SYS1_PLL_DCO] = &g12b_sys1_pll_dco.hw, + [CLKID_SYS1_PLL] = &g12b_sys1_pll.hw, + [CLKID_SYS1_PLL_DIV16_EN] = &g12b_sys1_pll_div16_en.hw, + [CLKID_SYS1_PLL_DIV16] = &g12b_sys1_pll_div16.hw, + [CLKID_CPUB_CLK_DYN0_SEL] = &g12b_cpub_clk_premux0.hw, + [CLKID_CPUB_CLK_DYN0_DIV] = &g12b_cpub_clk_mux0_div.hw, + [CLKID_CPUB_CLK_DYN0] = &g12b_cpub_clk_postmux0.hw, + [CLKID_CPUB_CLK_DYN1_SEL] = &g12b_cpub_clk_premux1.hw, + [CLKID_CPUB_CLK_DYN1_DIV] = &g12b_cpub_clk_mux1_div.hw, + [CLKID_CPUB_CLK_DYN1] = &g12b_cpub_clk_postmux1.hw, + [CLKID_CPUB_CLK_DYN] = &g12b_cpub_clk_dyn.hw, + [CLKID_CPUB_CLK] = &g12b_cpub_clk.hw, + [CLKID_CPUB_CLK_DIV16_EN] = &g12b_cpub_clk_div16_en.hw, + [CLKID_CPUB_CLK_DIV16] = &g12b_cpub_clk_div16.hw, + [CLKID_CPUB_CLK_DIV2] = &g12b_cpub_clk_div2.hw, + [CLKID_CPUB_CLK_DIV3] = &g12b_cpub_clk_div3.hw, + [CLKID_CPUB_CLK_DIV4] = &g12b_cpub_clk_div4.hw, + [CLKID_CPUB_CLK_DIV5] = &g12b_cpub_clk_div5.hw, + [CLKID_CPUB_CLK_DIV6] = &g12b_cpub_clk_div6.hw, + [CLKID_CPUB_CLK_DIV7] = &g12b_cpub_clk_div7.hw, + [CLKID_CPUB_CLK_DIV8] = &g12b_cpub_clk_div8.hw, + [CLKID_CPUB_CLK_APB_SEL] = &g12b_cpub_clk_apb_sel.hw, + [CLKID_CPUB_CLK_APB] = &g12b_cpub_clk_apb.hw, + [CLKID_CPUB_CLK_ATB_SEL] = &g12b_cpub_clk_atb_sel.hw, + [CLKID_CPUB_CLK_ATB] = &g12b_cpub_clk_atb.hw, + [CLKID_CPUB_CLK_AXI_SEL] = &g12b_cpub_clk_axi_sel.hw, + [CLKID_CPUB_CLK_AXI] = &g12b_cpub_clk_axi.hw, + [CLKID_CPUB_CLK_TRACE_SEL] = &g12b_cpub_clk_trace_sel.hw, + [CLKID_CPUB_CLK_TRACE] = &g12b_cpub_clk_trace.hw, [NR_CLKS] = NULL, }, .num = NR_CLKS, @@ -2966,16 +3763,52 @@ static struct clk_regmap *const g12a_clk_regmaps[] = { &g12a_vdec_hevcf_sel, &g12a_vdec_hevcf_div, &g12a_vdec_hevcf, + &g12a_ts_div, + &g12a_ts, + &g12b_cpu_clk, + &g12b_sys1_pll_dco, + &g12b_sys1_pll, + &g12b_sys1_pll_div16_en, + &g12b_cpub_clk_premux0, + &g12b_cpub_clk_mux0_div, + &g12b_cpub_clk_postmux0, + &g12b_cpub_clk_premux1, + &g12b_cpub_clk_mux1_div, + &g12b_cpub_clk_postmux1, + &g12b_cpub_clk_dyn, + &g12b_cpub_clk, + &g12b_cpub_clk_div16_en, + &g12b_cpub_clk_apb_sel, + &g12b_cpub_clk_apb, + &g12b_cpub_clk_atb_sel, + &g12b_cpub_clk_atb, + &g12b_cpub_clk_axi_sel, + &g12b_cpub_clk_axi, + &g12b_cpub_clk_trace_sel, + &g12b_cpub_clk_trace, +}; + +static const struct reg_sequence g12a_init_regs[] = { + { .reg = HHI_MPLL_CNTL0, .def = 0x00000543 }, }; static const struct meson_eeclkc_data g12a_clkc_data = { .regmap_clks = g12a_clk_regmaps, .regmap_clk_num = ARRAY_SIZE(g12a_clk_regmaps), - .hw_onecell_data = &g12a_hw_onecell_data + .hw_onecell_data = &g12a_hw_onecell_data, + .init_regs = g12a_init_regs, + .init_count = ARRAY_SIZE(g12a_init_regs), +}; + +static const struct meson_eeclkc_data g12b_clkc_data = { + .regmap_clks = g12a_clk_regmaps, + .regmap_clk_num = ARRAY_SIZE(g12a_clk_regmaps), + .hw_onecell_data = &g12b_hw_onecell_data }; static const struct of_device_id clkc_match_table[] = { { .compatible = "amlogic,g12a-clkc", .data = &g12a_clkc_data }, + { .compatible = "amlogic,g12b-clkc", .data = &g12b_clkc_data }, {} }; diff --git a/drivers/clk/meson/g12a.h b/drivers/clk/meson/g12a.h index bcc05cd9882f..c8aed31fbe17 100644 --- a/drivers/clk/meson/g12a.h +++ b/drivers/clk/meson/g12a.h @@ -69,6 +69,8 @@ #define HHI_VDEC4_CLK_CNTL 0x1EC #define HHI_HDCP22_CLK_CNTL 0x1F0 #define HHI_VAPBCLK_CNTL 0x1F4 +#define HHI_SYS_CPUB_CLK_CNTL1 0x200 +#define HHI_SYS_CPUB_CLK_CNTL 0x208 #define HHI_VPU_CLKB_CNTL 0x20C #define HHI_GEN_CLK_CNTL 0x228 #define HHI_VDIN_MEAS_CLK_CNTL 0x250 @@ -102,6 +104,13 @@ #define HHI_HDMI_PLL_CNTL5 0x334 #define HHI_HDMI_PLL_CNTL6 0x338 #define HHI_SPICC_CLK_CNTL 0x3dc +#define HHI_SYS1_PLL_CNTL0 0x380 +#define HHI_SYS1_PLL_CNTL1 0x384 +#define HHI_SYS1_PLL_CNTL2 0x388 +#define HHI_SYS1_PLL_CNTL3 0x38c +#define HHI_SYS1_PLL_CNTL4 0x390 +#define HHI_SYS1_PLL_CNTL5 0x394 +#define HHI_SYS1_PLL_CNTL6 0x398 /* * CLKID index values @@ -195,8 +204,38 @@ #define CLKID_VDEC_HEVC_DIV 206 #define CLKID_VDEC_HEVCF_SEL 208 #define CLKID_VDEC_HEVCF_DIV 209 +#define CLKID_TS_DIV 211 +#define CLKID_SYS1_PLL_DCO 213 +#define CLKID_SYS1_PLL 214 +#define CLKID_SYS1_PLL_DIV16_EN 215 +#define CLKID_SYS1_PLL_DIV16 216 +#define CLKID_CPUB_CLK_DYN0_SEL 217 +#define CLKID_CPUB_CLK_DYN0_DIV 218 +#define CLKID_CPUB_CLK_DYN0 219 +#define CLKID_CPUB_CLK_DYN1_SEL 220 +#define CLKID_CPUB_CLK_DYN1_DIV 221 +#define CLKID_CPUB_CLK_DYN1 222 +#define CLKID_CPUB_CLK_DYN 223 +#define CLKID_CPUB_CLK 224 +#define CLKID_CPUB_CLK_DIV16_EN 225 +#define CLKID_CPUB_CLK_DIV16 226 +#define CLKID_CPUB_CLK_DIV2 227 +#define CLKID_CPUB_CLK_DIV3 228 +#define CLKID_CPUB_CLK_DIV4 229 +#define CLKID_CPUB_CLK_DIV5 230 +#define CLKID_CPUB_CLK_DIV6 231 +#define CLKID_CPUB_CLK_DIV7 232 +#define CLKID_CPUB_CLK_DIV8 233 +#define CLKID_CPUB_CLK_APB_SEL 234 +#define CLKID_CPUB_CLK_APB 235 +#define CLKID_CPUB_CLK_ATB_SEL 236 +#define CLKID_CPUB_CLK_ATB 237 +#define CLKID_CPUB_CLK_AXI_SEL 238 +#define CLKID_CPUB_CLK_AXI 239 +#define CLKID_CPUB_CLK_TRACE_SEL 240 +#define CLKID_CPUB_CLK_TRACE 241 -#define NR_CLKS 211 +#define NR_CLKS 242 /* include the CLKIDs that have been made part of the DT binding */ #include <dt-bindings/clock/g12a-clkc.h> diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 29ffb4fde714..dab16d9b1af8 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -679,11 +679,6 @@ static struct clk_regmap gxbb_mpll0_div = { .shift = 16, .width = 9, }, - .ssen = { - .reg_off = HHI_MPLL_CNTL, - .shift = 25, - .width = 1, - }, .lock = &meson_clk_lock, }, .hw.init = &(struct clk_init_data){ diff --git a/drivers/clk/meson/meson-eeclk.c b/drivers/clk/meson/meson-eeclk.c index 37a34c9c3885..6ba2094be257 100644 --- a/drivers/clk/meson/meson-eeclk.c +++ b/drivers/clk/meson/meson-eeclk.c @@ -34,6 +34,9 @@ int meson_eeclkc_probe(struct platform_device *pdev) return PTR_ERR(map); } + if (data->init_count) + regmap_multi_reg_write(map, data->init_regs, data->init_count); + input = meson_clk_hw_register_input(dev, "xtal", IN_PREFIX "xtal", 0); if (IS_ERR(input)) { ret = PTR_ERR(input); diff --git a/drivers/clk/meson/meson-eeclk.h b/drivers/clk/meson/meson-eeclk.h index 1b809b1419fe..9ab5d6fa7ccb 100644 --- a/drivers/clk/meson/meson-eeclk.h +++ b/drivers/clk/meson/meson-eeclk.h @@ -17,6 +17,8 @@ struct platform_device; struct meson_eeclkc_data { struct clk_regmap *const *regmap_clks; unsigned int regmap_clk_num; + const struct reg_sequence *init_regs; + unsigned int init_count; struct clk_hw_onecell_data *hw_onecell_data; }; diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c index 62cd3a7f1f65..537219fa573e 100644 --- a/drivers/clk/meson/meson8b.c +++ b/drivers/clk/meson/meson8b.c @@ -2153,6 +2153,132 @@ static struct clk_regmap meson8b_vdec_hevc = { }, }; +/* TODO: the clock at index 0 is "DDR_PLL" which we don't support yet */ +static const char * const meson8b_cts_amclk_parent_names[] = { + "mpll0", "mpll1", "mpll2" +}; + +static u32 meson8b_cts_amclk_mux_table[] = { 1, 2, 3 }; + +static struct clk_regmap meson8b_cts_amclk_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_AUD_CLK_CNTL, + .mask = 0x3, + .shift = 9, + .table = meson8b_cts_amclk_mux_table, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_amclk_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = meson8b_cts_amclk_parent_names, + .num_parents = ARRAY_SIZE(meson8b_cts_amclk_parent_names), + }, +}; + +static struct clk_regmap meson8b_cts_amclk_div = { + .data = &(struct clk_regmap_div_data) { + .offset = HHI_AUD_CLK_CNTL, + .shift = 0, + .width = 8, + .flags = CLK_DIVIDER_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_amclk_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "cts_amclk_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap meson8b_cts_amclk = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_AUD_CLK_CNTL, + .bit_idx = 8, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_amclk", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_amclk_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +/* TODO: the clock at index 0 is "DDR_PLL" which we don't support yet */ +static const char * const meson8b_cts_mclk_i958_parent_names[] = { + "mpll0", "mpll1", "mpll2" +}; + +static u32 meson8b_cts_mclk_i958_mux_table[] = { 1, 2, 3 }; + +static struct clk_regmap meson8b_cts_mclk_i958_sel = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_AUD_CLK_CNTL2, + .mask = 0x3, + .shift = 25, + .table = meson8b_cts_mclk_i958_mux_table, + .flags = CLK_MUX_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data) { + .name = "cts_mclk_i958_sel", + .ops = &clk_regmap_mux_ops, + .parent_names = meson8b_cts_mclk_i958_parent_names, + .num_parents = ARRAY_SIZE(meson8b_cts_mclk_i958_parent_names), + }, +}; + +static struct clk_regmap meson8b_cts_mclk_i958_div = { + .data = &(struct clk_regmap_div_data){ + .offset = HHI_AUD_CLK_CNTL2, + .shift = 16, + .width = 8, + .flags = CLK_DIVIDER_ROUND_CLOSEST, + }, + .hw.init = &(struct clk_init_data) { + .name = "cts_mclk_i958_div", + .ops = &clk_regmap_divider_ops, + .parent_names = (const char *[]){ "cts_mclk_i958_sel" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap meson8b_cts_mclk_i958 = { + .data = &(struct clk_regmap_gate_data){ + .offset = HHI_AUD_CLK_CNTL2, + .bit_idx = 24, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_mclk_i958", + .ops = &clk_regmap_gate_ops, + .parent_names = (const char *[]){ "cts_mclk_i958_div" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_regmap meson8b_cts_i958 = { + .data = &(struct clk_regmap_mux_data){ + .offset = HHI_AUD_CLK_CNTL2, + .mask = 0x1, + .shift = 27, + }, + .hw.init = &(struct clk_init_data){ + .name = "cts_i958", + .ops = &clk_regmap_mux_ops, + .parent_names = (const char *[]){ "cts_amclk", + "cts_mclk_i958" }, + .num_parents = 2, + /* + * The parent is specific to origin of the audio data. Let the + * consumer choose the appropriate parent. + */ + .flags = CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + }, +}; + /* Everything Else (EE) domain gates */ static MESON_GATE(meson8b_ddr, HHI_GCLK_MPEG0, 0); @@ -2432,6 +2558,13 @@ static struct clk_hw_onecell_data meson8_hw_onecell_data = { [CLKID_VDEC_HEVC_DIV] = &meson8b_vdec_hevc_div.hw, [CLKID_VDEC_HEVC_EN] = &meson8b_vdec_hevc_en.hw, [CLKID_VDEC_HEVC] = &meson8b_vdec_hevc.hw, + [CLKID_CTS_AMCLK_SEL] = &meson8b_cts_amclk_sel.hw, + [CLKID_CTS_AMCLK_DIV] = &meson8b_cts_amclk_div.hw, + [CLKID_CTS_AMCLK] = &meson8b_cts_amclk.hw, + [CLKID_CTS_MCLK_I958_SEL] = &meson8b_cts_mclk_i958_sel.hw, + [CLKID_CTS_MCLK_I958_DIV] = &meson8b_cts_mclk_i958_div.hw, + [CLKID_CTS_MCLK_I958] = &meson8b_cts_mclk_i958.hw, + [CLKID_CTS_I958] = &meson8b_cts_i958.hw, [CLK_NR_CLKS] = NULL, }, .num = CLK_NR_CLKS, @@ -2641,6 +2774,13 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = { [CLKID_VDEC_HEVC_DIV] = &meson8b_vdec_hevc_div.hw, [CLKID_VDEC_HEVC_EN] = &meson8b_vdec_hevc_en.hw, [CLKID_VDEC_HEVC] = &meson8b_vdec_hevc.hw, + [CLKID_CTS_AMCLK_SEL] = &meson8b_cts_amclk_sel.hw, + [CLKID_CTS_AMCLK_DIV] = &meson8b_cts_amclk_div.hw, + [CLKID_CTS_AMCLK] = &meson8b_cts_amclk.hw, + [CLKID_CTS_MCLK_I958_SEL] = &meson8b_cts_mclk_i958_sel.hw, + [CLKID_CTS_MCLK_I958_DIV] = &meson8b_cts_mclk_i958_div.hw, + [CLKID_CTS_MCLK_I958] = &meson8b_cts_mclk_i958.hw, + [CLKID_CTS_I958] = &meson8b_cts_i958.hw, [CLK_NR_CLKS] = NULL, }, .num = CLK_NR_CLKS, @@ -2852,6 +2992,13 @@ static struct clk_hw_onecell_data meson8m2_hw_onecell_data = { [CLKID_VDEC_HEVC_DIV] = &meson8b_vdec_hevc_div.hw, [CLKID_VDEC_HEVC_EN] = &meson8b_vdec_hevc_en.hw, [CLKID_VDEC_HEVC] = &meson8b_vdec_hevc.hw, + [CLKID_CTS_AMCLK_SEL] = &meson8b_cts_amclk_sel.hw, + [CLKID_CTS_AMCLK_DIV] = &meson8b_cts_amclk_div.hw, + [CLKID_CTS_AMCLK] = &meson8b_cts_amclk.hw, + [CLKID_CTS_MCLK_I958_SEL] = &meson8b_cts_mclk_i958_sel.hw, + [CLKID_CTS_MCLK_I958_DIV] = &meson8b_cts_mclk_i958_div.hw, + [CLKID_CTS_MCLK_I958] = &meson8b_cts_mclk_i958.hw, + [CLKID_CTS_I958] = &meson8b_cts_i958.hw, [CLK_NR_CLKS] = NULL, }, .num = CLK_NR_CLKS, @@ -3041,6 +3188,13 @@ static struct clk_regmap *const meson8b_clk_regmaps[] = { &meson8b_vdec_hevc_div, &meson8b_vdec_hevc_en, &meson8b_vdec_hevc, + &meson8b_cts_amclk, + &meson8b_cts_amclk_sel, + &meson8b_cts_amclk_div, + &meson8b_cts_mclk_i958_sel, + &meson8b_cts_mclk_i958_div, + &meson8b_cts_mclk_i958, + &meson8b_cts_i958, }; static const struct meson8b_clk_reset_line { diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h index ed37196187e6..c889fbeec30f 100644 --- a/drivers/clk/meson/meson8b.h +++ b/drivers/clk/meson/meson8b.h @@ -30,7 +30,9 @@ #define HHI_SYS_CPU_CLK_CNTL1 0x15c /* 0x57 offset in data sheet */ #define HHI_VID_CLK_DIV 0x164 /* 0x59 offset in data sheet */ #define HHI_MPEG_CLK_CNTL 0x174 /* 0x5d offset in data sheet */ +#define HHI_AUD_CLK_CNTL 0x178 /* 0x5e offset in data sheet */ #define HHI_VID_CLK_CNTL 0x17c /* 0x5f offset in data sheet */ +#define HHI_AUD_CLK_CNTL2 0x190 /* 0x64 offset in data sheet */ #define HHI_VID_CLK_CNTL2 0x194 /* 0x65 offset in data sheet */ #define HHI_VID_DIVIDER_CNTL 0x198 /* 0x66 offset in data sheet */ #define HHI_SYS_CPU_CLK_CNTL0 0x19c /* 0x67 offset in data sheet */ @@ -171,8 +173,12 @@ #define CLKID_VDEC_HEVC_SEL 203 #define CLKID_VDEC_HEVC_DIV 204 #define CLKID_VDEC_HEVC_EN 205 +#define CLKID_CTS_AMCLK_SEL 207 +#define CLKID_CTS_AMCLK_DIV 208 +#define CLKID_CTS_MCLK_I958_SEL 210 +#define CLKID_CTS_MCLK_I958_DIV 211 -#define CLK_NR_CLKS 207 +#define CLK_NR_CLKS 214 /* * include the CLKID and RESETID that have diff --git a/drivers/clk/mmp/clk-frac.c b/drivers/clk/mmp/clk-frac.c index cb43d54735b0..90bf181f191a 100644 --- a/drivers/clk/mmp/clk-frac.c +++ b/drivers/clk/mmp/clk-frac.c @@ -78,11 +78,10 @@ static int clk_factor_set_rate(struct clk_hw *hw, unsigned long drate, struct mmp_clk_factor_masks *masks = factor->masks; int i; unsigned long val; - unsigned long prev_rate, rate = 0; + unsigned long rate = 0; unsigned long flags = 0; for (i = 0; i < factor->ftbl_cnt; i++) { - prev_rate = rate; rate = (((prate / 10000) * factor->ftbl[i].den) / (factor->ftbl[i].num * factor->masks->factor)) * 10000; if (rate > drate) diff --git a/drivers/clk/mvebu/kirkwood.c b/drivers/clk/mvebu/kirkwood.c index 35af3aa18f1c..47680237d0be 100644 --- a/drivers/clk/mvebu/kirkwood.c +++ b/drivers/clk/mvebu/kirkwood.c @@ -185,6 +185,11 @@ static void __init mv88f6180_get_clk_ratio( } } +static u32 __init mv98dx1135_get_tclk_freq(void __iomem *sar) +{ + return 166666667; +} + static const struct coreclk_soc_desc kirkwood_coreclks = { .get_tclk_freq = kirkwood_get_tclk_freq, .get_cpu_freq = kirkwood_get_cpu_freq, @@ -201,6 +206,14 @@ static const struct coreclk_soc_desc mv88f6180_coreclks = { .num_ratios = ARRAY_SIZE(kirkwood_coreclk_ratios), }; +static const struct coreclk_soc_desc mv98dx1135_coreclks = { + .get_tclk_freq = mv98dx1135_get_tclk_freq, + .get_cpu_freq = kirkwood_get_cpu_freq, + .get_clk_ratio = kirkwood_get_clk_ratio, + .ratios = kirkwood_coreclk_ratios, + .num_ratios = ARRAY_SIZE(kirkwood_coreclk_ratios), +}; + /* * Clock Gating Control */ @@ -325,6 +338,8 @@ static void __init kirkwood_clk_init(struct device_node *np) if (of_device_is_compatible(np, "marvell,mv88f6180-core-clock")) mvebu_coreclk_setup(np, &mv88f6180_coreclks); + else if (of_device_is_compatible(np, "marvell,mv98dx1135-core-clock")) + mvebu_coreclk_setup(np, &mv98dx1135_coreclks); else mvebu_coreclk_setup(np, &kirkwood_coreclks); @@ -339,3 +354,5 @@ CLK_OF_DECLARE(kirkwood_clk, "marvell,kirkwood-core-clock", kirkwood_clk_init); CLK_OF_DECLARE(mv88f6180_clk, "marvell,mv88f6180-core-clock", kirkwood_clk_init); +CLK_OF_DECLARE(98dx1135_clk, "marvell,mv98dx1135-core-clock", + kirkwood_clk_init); diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c index d2f39a972cad..d004cdaa0e39 100644 --- a/drivers/clk/qcom/gcc-msm8996.c +++ b/drivers/clk/qcom/gcc-msm8996.c @@ -130,22 +130,6 @@ static const char * const gcc_xo_gpll0_gpll4_gpll0_early_div[] = { "gpll0_early_div" }; -static const struct parent_map gcc_xo_gpll0_gpll2_gpll3_gpll0_early_div_map[] = { - { P_XO, 0 }, - { P_GPLL0, 1 }, - { P_GPLL2, 2 }, - { P_GPLL3, 3 }, - { P_GPLL0_EARLY_DIV, 6 } -}; - -static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll0_early_div[] = { - "xo", - "gpll0", - "gpll2", - "gpll3", - "gpll0_early_div" -}; - static const struct parent_map gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div_map[] = { { P_XO, 0 }, { P_GPLL0, 1 }, @@ -184,26 +168,6 @@ static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll2_early_gpll0_early "gpll0_early_div" }; -static const struct parent_map gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll4_gpll0_early_div_map[] = { - { P_XO, 0 }, - { P_GPLL0, 1 }, - { P_GPLL2, 2 }, - { P_GPLL3, 3 }, - { P_GPLL1, 4 }, - { P_GPLL4, 5 }, - { P_GPLL0_EARLY_DIV, 6 } -}; - -static const char * const gcc_xo_gpll0_gpll2_gpll3_gpll1_gpll4_gpll0_early_div[] = { - "xo", - "gpll0", - "gpll2", - "gpll3", - "gpll1", - "gpll4", - "gpll0_early_div" -}; - static struct clk_fixed_factor xo = { .mult = 1, .div = 1, diff --git a/drivers/clk/qcom/gcc-qcs404.c b/drivers/clk/qcom/gcc-qcs404.c index a54807eb3b28..29cf464dd2c8 100644 --- a/drivers/clk/qcom/gcc-qcs404.c +++ b/drivers/clk/qcom/gcc-qcs404.c @@ -2766,6 +2766,13 @@ static const struct qcom_reset_map gcc_qcs404_resets[] = { [GCC_PCIE_0_PHY_BCR] = { 0x3e004 }, [GCC_PCIE_0_LINK_DOWN_BCR] = { 0x3e038 }, [GCC_PCIEPHY_0_PHY_BCR] = { 0x3e03c }, + [GCC_PCIE_0_AXI_MASTER_STICKY_ARES] = { 0x3e040, 6}, + [GCC_PCIE_0_AHB_ARES] = { 0x3e040, 5 }, + [GCC_PCIE_0_AXI_SLAVE_ARES] = { 0x3e040, 4 }, + [GCC_PCIE_0_AXI_MASTER_ARES] = { 0x3e040, 3 }, + [GCC_PCIE_0_CORE_STICKY_ARES] = { 0x3e040, 2 }, + [GCC_PCIE_0_SLEEP_ARES] = { 0x3e040, 1 }, + [GCC_PCIE_0_PIPE_ARES] = { 0x3e040, 0 }, [GCC_EMAC_BCR] = { 0x4e000 }, }; diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index 679bc7d8950a..a250f59708d8 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -141,7 +141,9 @@ static int gdsc_toggle_logic(struct gdsc *sc, enum gdsc_status status) udelay(1); } - return gdsc_poll_status(sc, status); + ret = gdsc_poll_status(sc, status); + WARN(ret, "%s status stuck at 'o%s'", sc->pd.name, status ? "ff" : "n"); + return ret; } static inline int gdsc_deassert_reset(struct gdsc *sc) diff --git a/drivers/clk/renesas/clk-div6.c b/drivers/clk/renesas/clk-div6.c index e98a9f5b3c90..5ca183e70166 100644 --- a/drivers/clk/renesas/clk-div6.c +++ b/drivers/clk/renesas/clk-div6.c @@ -30,8 +30,8 @@ * @div: divisor value (1-64) * @src_shift: Shift to access the register bits to select the parent clock * @src_width: Number of register bits to select the parent clock (may be 0) - * @parents: Array to map from valid parent clocks indices to hardware indices * @nb: Notifier block to save/restore clock state for system resume + * @parents: Array to map from valid parent clocks indices to hardware indices */ struct div6_clock { struct clk_hw hw; @@ -39,8 +39,8 @@ struct div6_clock { unsigned int div; u32 src_shift; u32 src_width; - u8 *parents; struct notifier_block nb; + u8 parents[]; }; #define to_div6_clock(_hw) container_of(_hw, struct div6_clock, hw) @@ -221,17 +221,10 @@ struct clk * __init cpg_div6_register(const char *name, struct clk *clk; unsigned int i; - clock = kzalloc(sizeof(*clock), GFP_KERNEL); + clock = kzalloc(struct_size(clock, parents, num_parents), GFP_KERNEL); if (!clock) return ERR_PTR(-ENOMEM); - clock->parents = kmalloc_array(num_parents, sizeof(*clock->parents), - GFP_KERNEL); - if (!clock->parents) { - clk = ERR_PTR(-ENOMEM); - goto free_clock; - } - clock->reg = reg; /* @@ -259,7 +252,7 @@ struct clk * __init cpg_div6_register(const char *name, pr_err("%s: invalid number of parents for DIV6 clock %s\n", __func__, name); clk = ERR_PTR(-EINVAL); - goto free_parents; + goto free_clock; } /* Filter out invalid parents */ @@ -282,7 +275,7 @@ struct clk * __init cpg_div6_register(const char *name, clk = clk_register(NULL, &clock->hw); if (IS_ERR(clk)) - goto free_parents; + goto free_clock; if (notifiers) { clock->nb.notifier_call = cpg_div6_clock_notifier_call; @@ -291,8 +284,6 @@ struct clk * __init cpg_div6_register(const char *name, return clk; -free_parents: - kfree(clock->parents); free_clock: kfree(clock); return clk; diff --git a/drivers/clk/renesas/clk-mstp.c b/drivers/clk/renesas/clk-mstp.c index 92ece221b0d4..2db9093546c6 100644 --- a/drivers/clk/renesas/clk-mstp.c +++ b/drivers/clk/renesas/clk-mstp.c @@ -30,11 +30,12 @@ /** * struct mstp_clock_group - MSTP gating clocks group * - * @data: clocks in this group + * @data: clock specifier translation for clocks in this group * @smstpcr: module stop control register * @mstpsr: module stop status register (optional) * @lock: protects writes to SMSTPCR * @width_8bit: registers are 8-bit, not 32-bit + * @clks: clocks in this group */ struct mstp_clock_group { struct clk_onecell_data data; @@ -42,6 +43,7 @@ struct mstp_clock_group { void __iomem *mstpsr; spinlock_t lock; bool width_8bit; + struct clk *clks[]; }; /** @@ -186,14 +188,13 @@ static void __init cpg_mstp_clocks_init(struct device_node *np) struct clk **clks; unsigned int i; - group = kzalloc(sizeof(*group), GFP_KERNEL); - clks = kmalloc_array(MSTP_MAX_CLOCKS, sizeof(*clks), GFP_KERNEL); - if (group == NULL || clks == NULL) { + group = kzalloc(struct_size(group, clks, MSTP_MAX_CLOCKS), GFP_KERNEL); + if (group == NULL) { kfree(group); - kfree(clks); return; } + clks = group->clks; spin_lock_init(&group->lock); group->data.clks = clks; @@ -203,7 +204,6 @@ static void __init cpg_mstp_clocks_init(struct device_node *np) if (group->smstpcr == NULL) { pr_err("%s: failed to remap SMSTPCR\n", __func__); kfree(group); - kfree(clks); return; } @@ -297,16 +297,12 @@ found: return PTR_ERR(clk); error = pm_clk_create(dev); - if (error) { - dev_err(dev, "pm_clk_create failed %d\n", error); + if (error) goto fail_put; - } error = pm_clk_add_clk(dev, clk); - if (error) { - dev_err(dev, "pm_clk_add_clk %pC failed %d\n", clk, error); + if (error) goto fail_destroy; - } return 0; diff --git a/drivers/clk/renesas/r8a774a1-cpg-mssr.c b/drivers/clk/renesas/r8a774a1-cpg-mssr.c index 76ed7d1bae36..e05bfa200480 100644 --- a/drivers/clk/renesas/r8a774a1-cpg-mssr.c +++ b/drivers/clk/renesas/r8a774a1-cpg-mssr.c @@ -113,6 +113,11 @@ static const struct cpg_core_clk r8a774a1_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a774a1_mod_clks[] __initconst = { + DEF_MOD("tmu4", 121, R8A774A1_CLK_S0D6), + DEF_MOD("tmu3", 122, R8A774A1_CLK_S3D2), + DEF_MOD("tmu2", 123, R8A774A1_CLK_S3D2), + DEF_MOD("tmu1", 124, R8A774A1_CLK_S3D2), + DEF_MOD("tmu0", 125, R8A774A1_CLK_CP), DEF_MOD("fdp1-0", 119, R8A774A1_CLK_S0D1), DEF_MOD("scif5", 202, R8A774A1_CLK_S3D4), DEF_MOD("scif4", 203, R8A774A1_CLK_S3D4), diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c index 9e9a6f2c31e8..fbc8c75f4314 100644 --- a/drivers/clk/renesas/r8a7795-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c @@ -138,6 +138,7 @@ static struct mssr_mod_clk r8a7795_mod_clks[] __initdata = { DEF_MOD("cmt2", 301, R8A7795_CLK_R), DEF_MOD("cmt1", 302, R8A7795_CLK_R), DEF_MOD("cmt0", 303, R8A7795_CLK_R), + DEF_MOD("tpu0", 304, R8A7795_CLK_S3D4), DEF_MOD("scif2", 310, R8A7795_CLK_S3D4), DEF_MOD("sdif3", 311, R8A7795_CLK_SD3), DEF_MOD("sdif2", 312, R8A7795_CLK_SD2), @@ -201,6 +202,10 @@ static struct mssr_mod_clk r8a7795_mod_clks[] __initdata = { DEF_MOD("ehci0", 703, R8A7795_CLK_S3D2), DEF_MOD("hsusb", 704, R8A7795_CLK_S3D2), DEF_MOD("hsusb3", 705, R8A7795_CLK_S3D2), + DEF_MOD("cmm3", 708, R8A7795_CLK_S2D1), + DEF_MOD("cmm2", 709, R8A7795_CLK_S2D1), + DEF_MOD("cmm1", 710, R8A7795_CLK_S2D1), + DEF_MOD("cmm0", 711, R8A7795_CLK_S2D1), DEF_MOD("csi21", 713, R8A7795_CLK_CSI0), /* ES1.x */ DEF_MOD("csi20", 714, R8A7795_CLK_CSI0), DEF_MOD("csi41", 715, R8A7795_CLK_CSI0), diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c index d8e9af5d9ae9..90cc6a102602 100644 --- a/drivers/clk/renesas/r8a7796-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c @@ -134,6 +134,7 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = { DEF_MOD("cmt2", 301, R8A7796_CLK_R), DEF_MOD("cmt1", 302, R8A7796_CLK_R), DEF_MOD("cmt0", 303, R8A7796_CLK_R), + DEF_MOD("tpu0", 304, R8A7796_CLK_S3D4), DEF_MOD("scif2", 310, R8A7796_CLK_S3D4), DEF_MOD("sdif3", 311, R8A7796_CLK_SD3), DEF_MOD("sdif2", 312, R8A7796_CLK_SD2), @@ -180,6 +181,9 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = { DEF_MOD("ehci1", 702, R8A7796_CLK_S3D2), DEF_MOD("ehci0", 703, R8A7796_CLK_S3D2), DEF_MOD("hsusb", 704, R8A7796_CLK_S3D2), + DEF_MOD("cmm2", 709, R8A7796_CLK_S2D1), + DEF_MOD("cmm1", 710, R8A7796_CLK_S2D1), + DEF_MOD("cmm0", 711, R8A7796_CLK_S2D1), DEF_MOD("csi20", 714, R8A7796_CLK_CSI0), DEF_MOD("csi40", 716, R8A7796_CLK_CSI0), DEF_MOD("du2", 722, R8A7796_CLK_S2D1), diff --git a/drivers/clk/renesas/r8a77965-cpg-mssr.c b/drivers/clk/renesas/r8a77965-cpg-mssr.c index 8f87e314d949..b4e8c5b7d515 100644 --- a/drivers/clk/renesas/r8a77965-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77965-cpg-mssr.c @@ -132,6 +132,7 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = { DEF_MOD("cmt2", 301, R8A77965_CLK_R), DEF_MOD("cmt1", 302, R8A77965_CLK_R), DEF_MOD("cmt0", 303, R8A77965_CLK_R), + DEF_MOD("tpu0", 304, R8A77965_CLK_S3D4), DEF_MOD("scif2", 310, R8A77965_CLK_S3D4), DEF_MOD("sdif3", 311, R8A77965_CLK_SD3), DEF_MOD("sdif2", 312, R8A77965_CLK_SD2), @@ -179,6 +180,9 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = { DEF_MOD("ehci1", 702, R8A77965_CLK_S3D2), DEF_MOD("ehci0", 703, R8A77965_CLK_S3D2), DEF_MOD("hsusb", 704, R8A77965_CLK_S3D2), + DEF_MOD("cmm3", 708, R8A77965_CLK_S2D1), + DEF_MOD("cmm1", 710, R8A77965_CLK_S2D1), + DEF_MOD("cmm0", 711, R8A77965_CLK_S2D1), DEF_MOD("csi20", 714, R8A77965_CLK_CSI0), DEF_MOD("csi40", 716, R8A77965_CLK_CSI0), DEF_MOD("du3", 721, R8A77965_CLK_S2D1), diff --git a/drivers/clk/renesas/r8a77990-cpg-mssr.c b/drivers/clk/renesas/r8a77990-cpg-mssr.c index 9570404baa58..ceabf55c21c2 100644 --- a/drivers/clk/renesas/r8a77990-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77990-cpg-mssr.c @@ -183,6 +183,8 @@ static const struct mssr_mod_clk r8a77990_mod_clks[] __initconst = { DEF_MOD("ehci0", 703, R8A77990_CLK_S3D2), DEF_MOD("hsusb", 704, R8A77990_CLK_S3D2), + DEF_MOD("cmm1", 710, R8A77990_CLK_S1D1), + DEF_MOD("cmm0", 711, R8A77990_CLK_S1D1), DEF_MOD("csi40", 716, R8A77990_CLK_CSI0), DEF_MOD("du1", 723, R8A77990_CLK_S1D1), DEF_MOD("du0", 724, R8A77990_CLK_S1D1), diff --git a/drivers/clk/renesas/r8a77995-cpg-mssr.c b/drivers/clk/renesas/r8a77995-cpg-mssr.c index 68707277b17b..962bb337f2e7 100644 --- a/drivers/clk/renesas/r8a77995-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77995-cpg-mssr.c @@ -146,6 +146,8 @@ static const struct mssr_mod_clk r8a77995_mod_clks[] __initconst = { DEF_MOD("vspbs", 627, R8A77995_CLK_S0D1), DEF_MOD("ehci0", 703, R8A77995_CLK_S3D2), DEF_MOD("hsusb", 704, R8A77995_CLK_S3D2), + DEF_MOD("cmm1", 710, R8A77995_CLK_S1D1), + DEF_MOD("cmm0", 711, R8A77995_CLK_S1D1), DEF_MOD("du1", 723, R8A77995_CLK_S1D1), DEF_MOD("du0", 724, R8A77995_CLK_S1D1), DEF_MOD("lvds", 727, R8A77995_CLK_S2D1), diff --git a/drivers/clk/renesas/r9a06g032-clocks.c b/drivers/clk/renesas/r9a06g032-clocks.c index 7d042183aa37..b33e1383efe3 100644 --- a/drivers/clk/renesas/r9a06g032-clocks.c +++ b/drivers/clk/renesas/r9a06g032-clocks.c @@ -17,6 +17,8 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/platform_device.h> +#include <linux/pm_clock.h> +#include <linux/pm_domain.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <dt-bindings/clock/r9a06g032-sysctrl.h> @@ -29,6 +31,7 @@ struct r9a06g032_gate { /* This is used to describe a clock for instantiation */ struct r9a06g032_clkdesc { const char *name; + uint32_t managed: 1; uint32_t type: 3; uint32_t index: 8; uint32_t source : 8; /* source index + 1 (0 == none) */ @@ -61,7 +64,11 @@ struct r9a06g032_clkdesc { #define D_GATE(_idx, _n, _src, ...) \ { .type = K_GATE, .index = R9A06G032_##_idx, \ .source = 1 + R9A06G032_##_src, .name = _n, \ - .gate = I_GATE(__VA_ARGS__), } + .gate = I_GATE(__VA_ARGS__) } +#define D_MODULE(_idx, _n, _src, ...) \ + { .type = K_GATE, .index = R9A06G032_##_idx, \ + .source = 1 + R9A06G032_##_src, .name = _n, \ + .managed = 1, .gate = I_GATE(__VA_ARGS__) } #define D_ROOT(_idx, _n, _mul, _div) \ { .type = K_FFC, .index = R9A06G032_##_idx, .name = _n, \ .div = _div, .mul = _mul } @@ -122,7 +129,7 @@ enum { K_GATE = 0, K_FFC, K_DIV, K_BITSEL, K_DUALGATE }; #define R9A06G032_CLOCK_COUNT (R9A06G032_UART_GROUP_34567 + 1) -static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = { +static const struct r9a06g032_clkdesc r9a06g032_clocks[] = { D_ROOT(CLKOUT, "clkout", 25, 1), D_ROOT(CLK_PLL_USB, "clk_pll_usb", 12, 10), D_FFC(CLKOUT_D10, "clkout_d10", CLKOUT, 10), @@ -171,7 +178,7 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = { D_GATE(CLK_P6_PG2, "clk_p6_pg2", DIV_P6_PG, 0x8a3, 0x8a4, 0x8a5, 0, 0xb61, 0, 0), D_GATE(CLK_P6_PG3, "clk_p6_pg3", DIV_P6_PG, 0x8a6, 0x8a7, 0x8a8, 0, 0xb62, 0, 0), D_GATE(CLK_P6_PG4, "clk_p6_pg4", DIV_P6_PG, 0x8a9, 0x8aa, 0x8ab, 0, 0xb63, 0, 0), - D_GATE(CLK_PCI_USB, "clk_pci_usb", CLKOUT_D40, 0xe6, 0, 0, 0, 0, 0, 0), + D_MODULE(CLK_PCI_USB, "clk_pci_usb", CLKOUT_D40, 0xe6, 0, 0, 0, 0, 0, 0), D_GATE(CLK_QSPI0, "clk_qspi0", DIV_QSPI0, 0x2a4, 0x2a5, 0, 0, 0, 0, 0), D_GATE(CLK_QSPI1, "clk_qspi1", DIV_QSPI1, 0x484, 0x485, 0, 0, 0, 0, 0), D_GATE(CLK_RGMII_REF, "clk_rgmii_ref", CLKOUT_D8, 0x340, 0, 0, 0, 0, 0, 0), @@ -188,17 +195,17 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = { D_GATE(CLK_SPI5, "clk_spi5", DIV_P4_PG, 0x822, 0x823, 0, 0, 0, 0, 0), D_GATE(CLK_SWITCH, "clk_switch", DIV_SWITCH, 0x982, 0x983, 0, 0, 0, 0, 0), D_DIV(DIV_MOTOR, "div_motor", CLKOUT_D5, 84, 2, 8), - D_GATE(HCLK_ECAT125, "hclk_ecat125", CLKOUT_D8, 0x400, 0x401, 0, 0x402, 0, 0x440, 0x441), - D_GATE(HCLK_PINCONFIG, "hclk_pinconfig", CLKOUT_D40, 0x740, 0x741, 0x742, 0, 0xae0, 0, 0), - D_GATE(HCLK_SERCOS, "hclk_sercos", CLKOUT_D10, 0x420, 0x422, 0, 0x421, 0, 0x460, 0x461), - D_GATE(HCLK_SGPIO2, "hclk_sgpio2", DIV_P5_PG, 0x8c3, 0x8c4, 0x8c5, 0, 0xb41, 0, 0), - D_GATE(HCLK_SGPIO3, "hclk_sgpio3", DIV_P5_PG, 0x8c6, 0x8c7, 0x8c8, 0, 0xb42, 0, 0), - D_GATE(HCLK_SGPIO4, "hclk_sgpio4", DIV_P5_PG, 0x8c9, 0x8ca, 0x8cb, 0, 0xb43, 0, 0), - D_GATE(HCLK_TIMER0, "hclk_timer0", CLKOUT_D40, 0x743, 0x744, 0x745, 0, 0xae1, 0, 0), - D_GATE(HCLK_TIMER1, "hclk_timer1", CLKOUT_D40, 0x746, 0x747, 0x748, 0, 0xae2, 0, 0), - D_GATE(HCLK_USBF, "hclk_usbf", CLKOUT_D8, 0xe3, 0, 0, 0xe4, 0, 0x102, 0x103), - D_GATE(HCLK_USBH, "hclk_usbh", CLKOUT_D8, 0xe0, 0xe1, 0, 0xe2, 0, 0x100, 0x101), - D_GATE(HCLK_USBPM, "hclk_usbpm", CLKOUT_D8, 0xe5, 0, 0, 0, 0, 0, 0), + D_MODULE(HCLK_ECAT125, "hclk_ecat125", CLKOUT_D8, 0x400, 0x401, 0, 0x402, 0, 0x440, 0x441), + D_MODULE(HCLK_PINCONFIG, "hclk_pinconfig", CLKOUT_D40, 0x740, 0x741, 0x742, 0, 0xae0, 0, 0), + D_MODULE(HCLK_SERCOS, "hclk_sercos", CLKOUT_D10, 0x420, 0x422, 0, 0x421, 0, 0x460, 0x461), + D_MODULE(HCLK_SGPIO2, "hclk_sgpio2", DIV_P5_PG, 0x8c3, 0x8c4, 0x8c5, 0, 0xb41, 0, 0), + D_MODULE(HCLK_SGPIO3, "hclk_sgpio3", DIV_P5_PG, 0x8c6, 0x8c7, 0x8c8, 0, 0xb42, 0, 0), + D_MODULE(HCLK_SGPIO4, "hclk_sgpio4", DIV_P5_PG, 0x8c9, 0x8ca, 0x8cb, 0, 0xb43, 0, 0), + D_MODULE(HCLK_TIMER0, "hclk_timer0", CLKOUT_D40, 0x743, 0x744, 0x745, 0, 0xae1, 0, 0), + D_MODULE(HCLK_TIMER1, "hclk_timer1", CLKOUT_D40, 0x746, 0x747, 0x748, 0, 0xae2, 0, 0), + D_MODULE(HCLK_USBF, "hclk_usbf", CLKOUT_D8, 0xe3, 0, 0, 0xe4, 0, 0x102, 0x103), + D_MODULE(HCLK_USBH, "hclk_usbh", CLKOUT_D8, 0xe0, 0xe1, 0, 0xe2, 0, 0x100, 0x101), + D_MODULE(HCLK_USBPM, "hclk_usbpm", CLKOUT_D8, 0xe5, 0, 0, 0, 0, 0, 0), D_GATE(CLK_48_PG_F, "clk_48_pg_f", CLK_48, 0x78c, 0x78d, 0, 0x78e, 0, 0xb04, 0xb05), D_GATE(CLK_48_PG4, "clk_48_pg4", CLK_48, 0x789, 0x78a, 0x78b, 0, 0xb03, 0, 0), D_FFC(CLK_DDRPHY_PLLCLK_D4, "clk_ddrphy_pllclk_d4", CLK_DDRPHY_PLLCLK, 4), @@ -208,13 +215,13 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = { D_FFC(CLK_REF_SYNC_D8, "clk_ref_sync_d8", CLK_REF_SYNC, 8), D_FFC(CLK_SERCOS100_D2, "clk_sercos100_d2", CLK_SERCOS100, 2), D_DIV(DIV_CA7, "div_ca7", CLK_REF_SYNC, 57, 1, 4, 1, 2, 4), - D_GATE(HCLK_CAN0, "hclk_can0", CLK_48, 0x783, 0x784, 0x785, 0, 0xb01, 0, 0), - D_GATE(HCLK_CAN1, "hclk_can1", CLK_48, 0x786, 0x787, 0x788, 0, 0xb02, 0, 0), - D_GATE(HCLK_DELTASIGMA, "hclk_deltasigma", DIV_MOTOR, 0x1ef, 0x1f0, 0x1f1, 0, 0, 0, 0), - D_GATE(HCLK_PWMPTO, "hclk_pwmpto", DIV_MOTOR, 0x1ec, 0x1ed, 0x1ee, 0, 0, 0, 0), - D_GATE(HCLK_RSV, "hclk_rsv", CLK_48, 0x780, 0x781, 0x782, 0, 0xb00, 0, 0), - D_GATE(HCLK_SGPIO0, "hclk_sgpio0", DIV_MOTOR, 0x1e0, 0x1e1, 0x1e2, 0, 0, 0, 0), - D_GATE(HCLK_SGPIO1, "hclk_sgpio1", DIV_MOTOR, 0x1e3, 0x1e4, 0x1e5, 0, 0, 0, 0), + D_MODULE(HCLK_CAN0, "hclk_can0", CLK_48, 0x783, 0x784, 0x785, 0, 0xb01, 0, 0), + D_MODULE(HCLK_CAN1, "hclk_can1", CLK_48, 0x786, 0x787, 0x788, 0, 0xb02, 0, 0), + D_MODULE(HCLK_DELTASIGMA, "hclk_deltasigma", DIV_MOTOR, 0x1ef, 0x1f0, 0x1f1, 0, 0, 0, 0), + D_MODULE(HCLK_PWMPTO, "hclk_pwmpto", DIV_MOTOR, 0x1ec, 0x1ed, 0x1ee, 0, 0, 0, 0), + D_MODULE(HCLK_RSV, "hclk_rsv", CLK_48, 0x780, 0x781, 0x782, 0, 0xb00, 0, 0), + D_MODULE(HCLK_SGPIO0, "hclk_sgpio0", DIV_MOTOR, 0x1e0, 0x1e1, 0x1e2, 0, 0, 0, 0), + D_MODULE(HCLK_SGPIO1, "hclk_sgpio1", DIV_MOTOR, 0x1e3, 0x1e4, 0x1e5, 0, 0, 0, 0), D_DIV(RTOS_MDC, "rtos_mdc", CLK_REF_SYNC, 100, 80, 640, 80, 160, 320, 640), D_GATE(CLK_CM3, "clk_cm3", CLK_REF_SYNC_D4, 0xba0, 0xba1, 0, 0xba2, 0, 0xbc0, 0xbc1), D_GATE(CLK_DDRC, "clk_ddrc", CLK_DDRPHY_PLLCLK_D4, 0x323, 0x324, 0, 0, 0, 0, 0), @@ -222,53 +229,53 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = { D_GATE(CLK_HSR50, "clk_hsr50", CLK_HSR100_D2, 0x484, 0x485, 0, 0, 0, 0, 0), D_GATE(CLK_HW_RTOS, "clk_hw_rtos", CLK_REF_SYNC_D4, 0xc60, 0xc61, 0, 0, 0, 0, 0), D_GATE(CLK_SERCOS50, "clk_sercos50", CLK_SERCOS100_D2, 0x424, 0x423, 0, 0, 0, 0, 0), - D_GATE(HCLK_ADC, "hclk_adc", CLK_REF_SYNC_D8, 0x1af, 0x1b0, 0x1b1, 0, 0, 0, 0), - D_GATE(HCLK_CM3, "hclk_cm3", CLK_REF_SYNC_D4, 0xc20, 0xc21, 0xc22, 0, 0, 0, 0), - D_GATE(HCLK_CRYPTO_EIP150, "hclk_crypto_eip150", CLK_REF_SYNC_D4, 0x123, 0x124, 0x125, 0, 0x142, 0, 0), - D_GATE(HCLK_CRYPTO_EIP93, "hclk_crypto_eip93", CLK_REF_SYNC_D4, 0x120, 0x121, 0, 0x122, 0, 0x140, 0x141), - D_GATE(HCLK_DDRC, "hclk_ddrc", CLK_REF_SYNC_D4, 0x320, 0x322, 0, 0x321, 0, 0x3a0, 0x3a1), - D_GATE(HCLK_DMA0, "hclk_dma0", CLK_REF_SYNC_D4, 0x260, 0x261, 0x262, 0x263, 0x2c0, 0x2c1, 0x2c2), - D_GATE(HCLK_DMA1, "hclk_dma1", CLK_REF_SYNC_D4, 0x264, 0x265, 0x266, 0x267, 0x2c3, 0x2c4, 0x2c5), - D_GATE(HCLK_GMAC0, "hclk_gmac0", CLK_REF_SYNC_D4, 0x360, 0x361, 0x362, 0x363, 0x3c0, 0x3c1, 0x3c2), - D_GATE(HCLK_GMAC1, "hclk_gmac1", CLK_REF_SYNC_D4, 0x380, 0x381, 0x382, 0x383, 0x3e0, 0x3e1, 0x3e2), - D_GATE(HCLK_GPIO0, "hclk_gpio0", CLK_REF_SYNC_D4, 0x212, 0x213, 0x214, 0, 0, 0, 0), - D_GATE(HCLK_GPIO1, "hclk_gpio1", CLK_REF_SYNC_D4, 0x215, 0x216, 0x217, 0, 0, 0, 0), - D_GATE(HCLK_GPIO2, "hclk_gpio2", CLK_REF_SYNC_D4, 0x229, 0x22a, 0x22b, 0, 0, 0, 0), - D_GATE(HCLK_HSR, "hclk_hsr", CLK_HSR100_D2, 0x480, 0x482, 0, 0x481, 0, 0x4c0, 0x4c1), - D_GATE(HCLK_I2C0, "hclk_i2c0", CLK_REF_SYNC_D8, 0x1a9, 0x1aa, 0x1ab, 0, 0, 0, 0), - D_GATE(HCLK_I2C1, "hclk_i2c1", CLK_REF_SYNC_D8, 0x1ac, 0x1ad, 0x1ae, 0, 0, 0, 0), - D_GATE(HCLK_LCD, "hclk_lcd", CLK_REF_SYNC_D4, 0x7a0, 0x7a1, 0x7a2, 0, 0xb20, 0, 0), - D_GATE(HCLK_MSEBI_M, "hclk_msebi_m", CLK_REF_SYNC_D4, 0x164, 0x165, 0x166, 0, 0x183, 0, 0), - D_GATE(HCLK_MSEBI_S, "hclk_msebi_s", CLK_REF_SYNC_D4, 0x160, 0x161, 0x162, 0x163, 0x180, 0x181, 0x182), - D_GATE(HCLK_NAND, "hclk_nand", CLK_REF_SYNC_D4, 0x280, 0x281, 0x282, 0x283, 0x2e0, 0x2e1, 0x2e2), - D_GATE(HCLK_PG_I, "hclk_pg_i", CLK_REF_SYNC_D4, 0x7ac, 0x7ad, 0, 0x7ae, 0, 0xb24, 0xb25), - D_GATE(HCLK_PG19, "hclk_pg19", CLK_REF_SYNC_D4, 0x22c, 0x22d, 0x22e, 0, 0, 0, 0), - D_GATE(HCLK_PG20, "hclk_pg20", CLK_REF_SYNC_D4, 0x22f, 0x230, 0x231, 0, 0, 0, 0), - D_GATE(HCLK_PG3, "hclk_pg3", CLK_REF_SYNC_D4, 0x7a6, 0x7a7, 0x7a8, 0, 0xb22, 0, 0), - D_GATE(HCLK_PG4, "hclk_pg4", CLK_REF_SYNC_D4, 0x7a9, 0x7aa, 0x7ab, 0, 0xb23, 0, 0), - D_GATE(HCLK_QSPI0, "hclk_qspi0", CLK_REF_SYNC_D4, 0x2a0, 0x2a1, 0x2a2, 0x2a3, 0x300, 0x301, 0x302), - D_GATE(HCLK_QSPI1, "hclk_qspi1", CLK_REF_SYNC_D4, 0x480, 0x481, 0x482, 0x483, 0x4c0, 0x4c1, 0x4c2), - D_GATE(HCLK_ROM, "hclk_rom", CLK_REF_SYNC_D4, 0xaa0, 0xaa1, 0xaa2, 0, 0xb80, 0, 0), - D_GATE(HCLK_RTC, "hclk_rtc", CLK_REF_SYNC_D8, 0xa00, 0, 0, 0, 0, 0, 0), - D_GATE(HCLK_SDIO0, "hclk_sdio0", CLK_REF_SYNC_D4, 0x60, 0x61, 0x62, 0x63, 0x80, 0x81, 0x82), - D_GATE(HCLK_SDIO1, "hclk_sdio1", CLK_REF_SYNC_D4, 0x640, 0x641, 0x642, 0x643, 0x660, 0x661, 0x662), - D_GATE(HCLK_SEMAP, "hclk_semap", CLK_REF_SYNC_D4, 0x7a3, 0x7a4, 0x7a5, 0, 0xb21, 0, 0), - D_GATE(HCLK_SPI0, "hclk_spi0", CLK_REF_SYNC_D4, 0x200, 0x201, 0x202, 0, 0, 0, 0), - D_GATE(HCLK_SPI1, "hclk_spi1", CLK_REF_SYNC_D4, 0x203, 0x204, 0x205, 0, 0, 0, 0), - D_GATE(HCLK_SPI2, "hclk_spi2", CLK_REF_SYNC_D4, 0x206, 0x207, 0x208, 0, 0, 0, 0), - D_GATE(HCLK_SPI3, "hclk_spi3", CLK_REF_SYNC_D4, 0x209, 0x20a, 0x20b, 0, 0, 0, 0), - D_GATE(HCLK_SPI4, "hclk_spi4", CLK_REF_SYNC_D4, 0x20c, 0x20d, 0x20e, 0, 0, 0, 0), - D_GATE(HCLK_SPI5, "hclk_spi5", CLK_REF_SYNC_D4, 0x20f, 0x210, 0x211, 0, 0, 0, 0), - D_GATE(HCLK_SWITCH, "hclk_switch", CLK_REF_SYNC_D4, 0x980, 0, 0x981, 0, 0, 0, 0), - D_GATE(HCLK_SWITCH_RG, "hclk_switch_rg", CLK_REF_SYNC_D4, 0xc40, 0xc41, 0xc42, 0, 0, 0, 0), - D_GATE(HCLK_UART0, "hclk_uart0", CLK_REF_SYNC_D8, 0x1a0, 0x1a1, 0x1a2, 0, 0, 0, 0), - D_GATE(HCLK_UART1, "hclk_uart1", CLK_REF_SYNC_D8, 0x1a3, 0x1a4, 0x1a5, 0, 0, 0, 0), - D_GATE(HCLK_UART2, "hclk_uart2", CLK_REF_SYNC_D8, 0x1a6, 0x1a7, 0x1a8, 0, 0, 0, 0), - D_GATE(HCLK_UART3, "hclk_uart3", CLK_REF_SYNC_D4, 0x218, 0x219, 0x21a, 0, 0, 0, 0), - D_GATE(HCLK_UART4, "hclk_uart4", CLK_REF_SYNC_D4, 0x21b, 0x21c, 0x21d, 0, 0, 0, 0), - D_GATE(HCLK_UART5, "hclk_uart5", CLK_REF_SYNC_D4, 0x220, 0x221, 0x222, 0, 0, 0, 0), - D_GATE(HCLK_UART6, "hclk_uart6", CLK_REF_SYNC_D4, 0x223, 0x224, 0x225, 0, 0, 0, 0), - D_GATE(HCLK_UART7, "hclk_uart7", CLK_REF_SYNC_D4, 0x226, 0x227, 0x228, 0, 0, 0, 0), + D_MODULE(HCLK_ADC, "hclk_adc", CLK_REF_SYNC_D8, 0x1af, 0x1b0, 0x1b1, 0, 0, 0, 0), + D_MODULE(HCLK_CM3, "hclk_cm3", CLK_REF_SYNC_D4, 0xc20, 0xc21, 0xc22, 0, 0, 0, 0), + D_MODULE(HCLK_CRYPTO_EIP150, "hclk_crypto_eip150", CLK_REF_SYNC_D4, 0x123, 0x124, 0x125, 0, 0x142, 0, 0), + D_MODULE(HCLK_CRYPTO_EIP93, "hclk_crypto_eip93", CLK_REF_SYNC_D4, 0x120, 0x121, 0, 0x122, 0, 0x140, 0x141), + D_MODULE(HCLK_DDRC, "hclk_ddrc", CLK_REF_SYNC_D4, 0x320, 0x322, 0, 0x321, 0, 0x3a0, 0x3a1), + D_MODULE(HCLK_DMA0, "hclk_dma0", CLK_REF_SYNC_D4, 0x260, 0x261, 0x262, 0x263, 0x2c0, 0x2c1, 0x2c2), + D_MODULE(HCLK_DMA1, "hclk_dma1", CLK_REF_SYNC_D4, 0x264, 0x265, 0x266, 0x267, 0x2c3, 0x2c4, 0x2c5), + D_MODULE(HCLK_GMAC0, "hclk_gmac0", CLK_REF_SYNC_D4, 0x360, 0x361, 0x362, 0x363, 0x3c0, 0x3c1, 0x3c2), + D_MODULE(HCLK_GMAC1, "hclk_gmac1", CLK_REF_SYNC_D4, 0x380, 0x381, 0x382, 0x383, 0x3e0, 0x3e1, 0x3e2), + D_MODULE(HCLK_GPIO0, "hclk_gpio0", CLK_REF_SYNC_D4, 0x212, 0x213, 0x214, 0, 0, 0, 0), + D_MODULE(HCLK_GPIO1, "hclk_gpio1", CLK_REF_SYNC_D4, 0x215, 0x216, 0x217, 0, 0, 0, 0), + D_MODULE(HCLK_GPIO2, "hclk_gpio2", CLK_REF_SYNC_D4, 0x229, 0x22a, 0x22b, 0, 0, 0, 0), + D_MODULE(HCLK_HSR, "hclk_hsr", CLK_HSR100_D2, 0x480, 0x482, 0, 0x481, 0, 0x4c0, 0x4c1), + D_MODULE(HCLK_I2C0, "hclk_i2c0", CLK_REF_SYNC_D8, 0x1a9, 0x1aa, 0x1ab, 0, 0, 0, 0), + D_MODULE(HCLK_I2C1, "hclk_i2c1", CLK_REF_SYNC_D8, 0x1ac, 0x1ad, 0x1ae, 0, 0, 0, 0), + D_MODULE(HCLK_LCD, "hclk_lcd", CLK_REF_SYNC_D4, 0x7a0, 0x7a1, 0x7a2, 0, 0xb20, 0, 0), + D_MODULE(HCLK_MSEBI_M, "hclk_msebi_m", CLK_REF_SYNC_D4, 0x164, 0x165, 0x166, 0, 0x183, 0, 0), + D_MODULE(HCLK_MSEBI_S, "hclk_msebi_s", CLK_REF_SYNC_D4, 0x160, 0x161, 0x162, 0x163, 0x180, 0x181, 0x182), + D_MODULE(HCLK_NAND, "hclk_nand", CLK_REF_SYNC_D4, 0x280, 0x281, 0x282, 0x283, 0x2e0, 0x2e1, 0x2e2), + D_MODULE(HCLK_PG_I, "hclk_pg_i", CLK_REF_SYNC_D4, 0x7ac, 0x7ad, 0, 0x7ae, 0, 0xb24, 0xb25), + D_MODULE(HCLK_PG19, "hclk_pg19", CLK_REF_SYNC_D4, 0x22c, 0x22d, 0x22e, 0, 0, 0, 0), + D_MODULE(HCLK_PG20, "hclk_pg20", CLK_REF_SYNC_D4, 0x22f, 0x230, 0x231, 0, 0, 0, 0), + D_MODULE(HCLK_PG3, "hclk_pg3", CLK_REF_SYNC_D4, 0x7a6, 0x7a7, 0x7a8, 0, 0xb22, 0, 0), + D_MODULE(HCLK_PG4, "hclk_pg4", CLK_REF_SYNC_D4, 0x7a9, 0x7aa, 0x7ab, 0, 0xb23, 0, 0), + D_MODULE(HCLK_QSPI0, "hclk_qspi0", CLK_REF_SYNC_D4, 0x2a0, 0x2a1, 0x2a2, 0x2a3, 0x300, 0x301, 0x302), + D_MODULE(HCLK_QSPI1, "hclk_qspi1", CLK_REF_SYNC_D4, 0x480, 0x481, 0x482, 0x483, 0x4c0, 0x4c1, 0x4c2), + D_MODULE(HCLK_ROM, "hclk_rom", CLK_REF_SYNC_D4, 0xaa0, 0xaa1, 0xaa2, 0, 0xb80, 0, 0), + D_MODULE(HCLK_RTC, "hclk_rtc", CLK_REF_SYNC_D8, 0xa00, 0, 0, 0, 0, 0, 0), + D_MODULE(HCLK_SDIO0, "hclk_sdio0", CLK_REF_SYNC_D4, 0x60, 0x61, 0x62, 0x63, 0x80, 0x81, 0x82), + D_MODULE(HCLK_SDIO1, "hclk_sdio1", CLK_REF_SYNC_D4, 0x640, 0x641, 0x642, 0x643, 0x660, 0x661, 0x662), + D_MODULE(HCLK_SEMAP, "hclk_semap", CLK_REF_SYNC_D4, 0x7a3, 0x7a4, 0x7a5, 0, 0xb21, 0, 0), + D_MODULE(HCLK_SPI0, "hclk_spi0", CLK_REF_SYNC_D4, 0x200, 0x201, 0x202, 0, 0, 0, 0), + D_MODULE(HCLK_SPI1, "hclk_spi1", CLK_REF_SYNC_D4, 0x203, 0x204, 0x205, 0, 0, 0, 0), + D_MODULE(HCLK_SPI2, "hclk_spi2", CLK_REF_SYNC_D4, 0x206, 0x207, 0x208, 0, 0, 0, 0), + D_MODULE(HCLK_SPI3, "hclk_spi3", CLK_REF_SYNC_D4, 0x209, 0x20a, 0x20b, 0, 0, 0, 0), + D_MODULE(HCLK_SPI4, "hclk_spi4", CLK_REF_SYNC_D4, 0x20c, 0x20d, 0x20e, 0, 0, 0, 0), + D_MODULE(HCLK_SPI5, "hclk_spi5", CLK_REF_SYNC_D4, 0x20f, 0x210, 0x211, 0, 0, 0, 0), + D_MODULE(HCLK_SWITCH, "hclk_switch", CLK_REF_SYNC_D4, 0x980, 0, 0x981, 0, 0, 0, 0), + D_MODULE(HCLK_SWITCH_RG, "hclk_switch_rg", CLK_REF_SYNC_D4, 0xc40, 0xc41, 0xc42, 0, 0, 0, 0), + D_MODULE(HCLK_UART0, "hclk_uart0", CLK_REF_SYNC_D8, 0x1a0, 0x1a1, 0x1a2, 0, 0, 0, 0), + D_MODULE(HCLK_UART1, "hclk_uart1", CLK_REF_SYNC_D8, 0x1a3, 0x1a4, 0x1a5, 0, 0, 0, 0), + D_MODULE(HCLK_UART2, "hclk_uart2", CLK_REF_SYNC_D8, 0x1a6, 0x1a7, 0x1a8, 0, 0, 0, 0), + D_MODULE(HCLK_UART3, "hclk_uart3", CLK_REF_SYNC_D4, 0x218, 0x219, 0x21a, 0, 0, 0, 0), + D_MODULE(HCLK_UART4, "hclk_uart4", CLK_REF_SYNC_D4, 0x21b, 0x21c, 0x21d, 0, 0, 0, 0), + D_MODULE(HCLK_UART5, "hclk_uart5", CLK_REF_SYNC_D4, 0x220, 0x221, 0x222, 0, 0, 0, 0), + D_MODULE(HCLK_UART6, "hclk_uart6", CLK_REF_SYNC_D4, 0x223, 0x224, 0x225, 0, 0, 0, 0), + D_MODULE(HCLK_UART7, "hclk_uart7", CLK_REF_SYNC_D4, 0x226, 0x227, 0x228, 0, 0, 0, 0), /* * These are not hardware clocks, but are needed to handle the special * case where we have a 'selector bit' that doesn't just change the @@ -345,6 +352,84 @@ struct r9a06g032_clk_gate { #define to_r9a06g032_gate(_hw) container_of(_hw, struct r9a06g032_clk_gate, hw) +static int create_add_module_clock(struct of_phandle_args *clkspec, + struct device *dev) +{ + struct clk *clk; + int error; + + clk = of_clk_get_from_provider(clkspec); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + error = pm_clk_create(dev); + if (error) { + clk_put(clk); + return error; + } + + error = pm_clk_add_clk(dev, clk); + if (error) { + pm_clk_destroy(dev); + clk_put(clk); + } + + return error; +} + +static int r9a06g032_attach_dev(struct generic_pm_domain *pd, + struct device *dev) +{ + struct device_node *np = dev->of_node; + struct of_phandle_args clkspec; + int i = 0; + int error; + int index; + + while (!of_parse_phandle_with_args(np, "clocks", "#clock-cells", i, + &clkspec)) { + if (clkspec.np != pd->dev.of_node) + continue; + + index = clkspec.args[0]; + if (index < R9A06G032_CLOCK_COUNT && + r9a06g032_clocks[index].managed) { + error = create_add_module_clock(&clkspec, dev); + of_node_put(clkspec.np); + if (error) + return error; + } + i++; + } + + return 0; +} + +static void r9a06g032_detach_dev(struct generic_pm_domain *unused, struct device *dev) +{ + if (!pm_clk_no_clocks(dev)) + pm_clk_destroy(dev); +} + +static int r9a06g032_add_clk_domain(struct device *dev) +{ + struct device_node *np = dev->of_node; + struct generic_pm_domain *pd; + + pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + pd->name = np->name; + pd->flags = GENPD_FLAG_PM_CLK | GENPD_FLAG_ACTIVE_WAKEUP; + pd->attach_dev = r9a06g032_attach_dev; + pd->detach_dev = r9a06g032_detach_dev; + pm_genpd_init(pd, &pm_domain_always_on_gov, false); + + of_genpd_add_provider_simple(np, pd); + return 0; +} + static void r9a06g032_clk_gate_set(struct r9a06g032_priv *clocks, struct r9a06g032_gate *g, int on) @@ -871,8 +956,12 @@ static int __init r9a06g032_clocks_probe(struct platform_device *pdev) if (error) return error; - return devm_add_action_or_reset(dev, + error = devm_add_action_or_reset(dev, r9a06g032_clocks_del_clk_provider, np); + if (error) + return error; + + return r9a06g032_add_clk_domain(dev); } static const struct of_device_id r9a06g032_match[] = { diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index 0201809bbd37..52bbb9ce3807 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -112,14 +112,15 @@ static const u16 srcr[] = { * @dev: CPG/MSSR device * @base: CPG/MSSR register block base address * @rmw_lock: protects RMW register accesses - * @clks: Array containing all Core and Module Clocks + * @np: Device node in DT for this CPG/MSSR module * @num_core_clks: Number of Core Clocks in clks[] * @num_mod_clks: Number of Module Clocks in clks[] * @last_dt_core_clk: ID of the last Core Clock exported to DT + * @stbyctrl: This device has Standby Control Registers * @notifiers: Notifier chain to save/restore clock state for system resume * @smstpcr_saved[].mask: Mask of SMSTPCR[] bits under our control * @smstpcr_saved[].val: Saved values of SMSTPCR[] - * @stbyctrl: This device has Standby Control Registers + * @clks: Array containing all Core and Module Clocks */ struct cpg_mssr_priv { #ifdef CONFIG_RESET_CONTROLLER @@ -130,7 +131,6 @@ struct cpg_mssr_priv { spinlock_t rmw_lock; struct device_node *np; - struct clk **clks; unsigned int num_core_clks; unsigned int num_mod_clks; unsigned int last_dt_core_clk; @@ -141,6 +141,8 @@ struct cpg_mssr_priv { u32 mask; u32 val; } smstpcr_saved[ARRAY_SIZE(smstpcr)]; + + struct clk *clks[]; }; static struct cpg_mssr_priv *cpg_mssr_priv; @@ -447,9 +449,8 @@ fail: struct cpg_mssr_clk_domain { struct generic_pm_domain genpd; - struct device_node *np; unsigned int num_core_pm_clks; - unsigned int core_pm_clks[0]; + unsigned int core_pm_clks[]; }; static struct cpg_mssr_clk_domain *cpg_mssr_clk_domain; @@ -459,7 +460,7 @@ static bool cpg_mssr_is_pm_clk(const struct of_phandle_args *clkspec, { unsigned int i; - if (clkspec->np != pd->np || clkspec->args_count != 2) + if (clkspec->np != pd->genpd.dev.of_node || clkspec->args_count != 2) return false; switch (clkspec->args[0]) { @@ -510,16 +511,12 @@ found: return PTR_ERR(clk); error = pm_clk_create(dev); - if (error) { - dev_err(dev, "pm_clk_create failed %d\n", error); + if (error) goto fail_put; - } error = pm_clk_add_clk(dev, clk); - if (error) { - dev_err(dev, "pm_clk_add_clk %pC failed %d\n", clk, error); + if (error) goto fail_destroy; - } return 0; @@ -549,7 +546,6 @@ static int __init cpg_mssr_add_clk_domain(struct device *dev, if (!pd) return -ENOMEM; - pd->np = np; pd->num_core_pm_clks = num_core_pm_clks; memcpy(pd->core_pm_clks, core_pm_clks, pm_size); @@ -896,7 +892,6 @@ static int __init cpg_mssr_common_init(struct device *dev, const struct cpg_mssr_info *info) { struct cpg_mssr_priv *priv; - struct clk **clks = NULL; unsigned int nclks, i; int error; @@ -906,7 +901,8 @@ static int __init cpg_mssr_common_init(struct device *dev, return error; } - priv = kzalloc(sizeof(*priv), GFP_KERNEL); + nclks = info->num_total_core_clks + info->num_hw_mod_clks; + priv = kzalloc(struct_size(priv, clks, nclks), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -920,15 +916,7 @@ static int __init cpg_mssr_common_init(struct device *dev, goto out_err; } - nclks = info->num_total_core_clks + info->num_hw_mod_clks; - clks = kmalloc_array(nclks, sizeof(*clks), GFP_KERNEL); - if (!clks) { - error = -ENOMEM; - goto out_err; - } - cpg_mssr_priv = priv; - priv->clks = clks; priv->num_core_clks = info->num_total_core_clks; priv->num_mod_clks = info->num_hw_mod_clks; priv->last_dt_core_clk = info->last_dt_core_clk; @@ -936,7 +924,7 @@ static int __init cpg_mssr_common_init(struct device *dev, priv->stbyctrl = info->stbyctrl; for (i = 0; i < nclks; i++) - clks[i] = ERR_PTR(-ENOENT); + priv->clks[i] = ERR_PTR(-ENOENT); error = of_clk_add_provider(np, cpg_mssr_clk_src_twocell_get, priv); if (error) @@ -945,7 +933,6 @@ static int __init cpg_mssr_common_init(struct device *dev, return 0; out_err: - kfree(clks); if (priv->base) iounmap(priv->base); kfree(priv); diff --git a/drivers/clk/rockchip/clk-mmc-phase.c b/drivers/clk/rockchip/clk-mmc-phase.c index c61f4d3e52e2..4abe7ff31f53 100644 --- a/drivers/clk/rockchip/clk-mmc-phase.c +++ b/drivers/clk/rockchip/clk-mmc-phase.c @@ -46,29 +46,27 @@ static unsigned long rockchip_mmc_recalc(struct clk_hw *hw, static int rockchip_mmc_get_phase(struct clk_hw *hw) { struct rockchip_mmc_clock *mmc_clock = to_mmc_clock(hw); - unsigned long rate = clk_get_rate(hw->clk); + unsigned long rate = clk_hw_get_rate(hw); u32 raw_value; u16 degrees; u32 delay_num = 0; /* See the comment for rockchip_mmc_set_phase below */ - if (!rate) { - pr_err("%s: invalid clk rate\n", __func__); + if (!rate) return -EINVAL; - } raw_value = readl(mmc_clock->reg) >> (mmc_clock->shift); degrees = (raw_value & ROCKCHIP_MMC_DEGREE_MASK) * 90; if (raw_value & ROCKCHIP_MMC_DELAY_SEL) { - /* degrees/delaynum * 10000 */ + /* degrees/delaynum * 1000000 */ unsigned long factor = (ROCKCHIP_MMC_DELAY_ELEMENT_PSEC / 10) * - 36 * (rate / 1000000); + 36 * (rate / 10000); delay_num = (raw_value & ROCKCHIP_MMC_DELAYNUM_MASK); delay_num >>= ROCKCHIP_MMC_DELAYNUM_OFFSET; - degrees += DIV_ROUND_CLOSEST(delay_num * factor, 10000); + degrees += DIV_ROUND_CLOSEST(delay_num * factor, 1000000); } return degrees % 360; @@ -77,7 +75,7 @@ static int rockchip_mmc_get_phase(struct clk_hw *hw) static int rockchip_mmc_set_phase(struct clk_hw *hw, int degrees) { struct rockchip_mmc_clock *mmc_clock = to_mmc_clock(hw); - unsigned long rate = clk_get_rate(hw->clk); + unsigned long rate = clk_hw_get_rate(hw); u8 nineties, remainder; u8 delay_num; u32 raw_value; diff --git a/drivers/clk/rockchip/clk-px30.c b/drivers/clk/rockchip/clk-px30.c index bb39a799bdb5..3a501896b280 100644 --- a/drivers/clk/rockchip/clk-px30.c +++ b/drivers/clk/rockchip/clk-px30.c @@ -794,6 +794,9 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { GATE(ACLK_GIC, "aclk_gic", "aclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 12, GFLAGS), GATE(ACLK_DCF, "aclk_dcf", "aclk_bus_pre", 0, PX30_CLKGATE_CON(13), 15, GFLAGS), + /* aclk_dmac is controlled by sgrf_soc_con1[11]. */ + SGRF_GATE(ACLK_DMAC, "aclk_dmac", "aclk_bus_pre"), + GATE(0, "hclk_bus_niu", "hclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 9, GFLAGS), GATE(0, "hclk_rom", "hclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 14, GFLAGS), GATE(HCLK_PDM, "hclk_pdm", "hclk_bus_pre", 0, PX30_CLKGATE_CON(14), 1, GFLAGS), @@ -957,7 +960,6 @@ static void __init px30_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; - struct clk *clk; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -972,14 +974,6 @@ static void __init px30_clk_init(struct device_node *np) return; } - /* aclk_dmac is controlled by sgrf_soc_con1[11]. */ - clk = clk_register_fixed_factor(NULL, "aclk_dmac", "aclk_bus_pre", 0, 1, 1); - if (IS_ERR(clk)) - pr_warn("%s: could not register clock aclk_dmac: %ld\n", - __func__, PTR_ERR(clk)); - else - rockchip_clk_add_lookup(ctx, clk, ACLK_DMAC); - rockchip_clk_register_plls(ctx, px30_pll_clks, ARRAY_SIZE(px30_pll_clks), PX30_GRF_SOC_STATUS0); diff --git a/drivers/clk/rockchip/clk-rk3228.c b/drivers/clk/rockchip/clk-rk3228.c index bdb126321e56..d17cfb7a3ff4 100644 --- a/drivers/clk/rockchip/clk-rk3228.c +++ b/drivers/clk/rockchip/clk-rk3228.c @@ -101,6 +101,7 @@ static struct rockchip_cpuclk_rate_table rk3228_cpuclk_rates[] __initdata = { RK3228_CPUCLK_RATE(1608000000, 1, 7), RK3228_CPUCLK_RATE(1512000000, 1, 7), RK3228_CPUCLK_RATE(1488000000, 1, 5), + RK3228_CPUCLK_RATE(1464000000, 1, 5), RK3228_CPUCLK_RATE(1416000000, 1, 5), RK3228_CPUCLK_RATE(1392000000, 1, 5), RK3228_CPUCLK_RATE(1296000000, 1, 5), @@ -246,7 +247,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { RK2928_CLKGATE_CON(4), 0, GFLAGS), /* PD_MISC */ - MUX(0, "hdmiphy", mux_hdmiphy_p, CLK_SET_RATE_PARENT, + MUX(SCLK_HDMI_PHY, "hdmiphy", mux_hdmiphy_p, CLK_SET_RATE_PARENT, RK2928_MISC_CON, 13, 1, MFLAGS), MUX(0, "usb480m_phy", mux_usb480m_phy_p, CLK_SET_RATE_PARENT, RK2928_MISC_CON, 14, 1, MFLAGS), diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c index 057629685ea1..cc2a177bbdbf 100644 --- a/drivers/clk/rockchip/clk-rk3288.c +++ b/drivers/clk/rockchip/clk-rk3288.c @@ -113,7 +113,6 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = { RK3066_PLL_RATE( 160000000, 1, 80, 12), RK3066_PLL_RATE( 157500000, 1, 105, 16), RK3066_PLL_RATE( 126000000, 1, 84, 16), - RK3066_PLL_RATE( 48000000, 1, 64, 32), { /* sentinel */ }, }; @@ -767,6 +766,9 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(14), 11, GFLAGS), GATE(0, "pclk_alive_niu", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 12, GFLAGS), + /* Watchdog pclk is controlled by RK3288_SGRF_SOC_CON0[1]. */ + SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_pd_alive"), + /* pclk_pd_pmu gates */ GATE(PCLK_PMU, "pclk_pmu", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 0, GFLAGS), GATE(0, "pclk_intmem1", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 1, GFLAGS), @@ -915,7 +917,6 @@ static struct syscore_ops rk3288_clk_syscore_ops = { static void __init rk3288_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; - struct clk *clk; rk3288_cru_base = of_iomap(np, 0); if (!rk3288_cru_base) { @@ -930,14 +931,6 @@ static void __init rk3288_clk_init(struct device_node *np) return; } - /* Watchdog pclk is controlled by RK3288_SGRF_SOC_CON0[1]. */ - clk = clk_register_fixed_factor(NULL, "pclk_wdt", "pclk_pd_alive", 0, 1, 1); - if (IS_ERR(clk)) - pr_warn("%s: could not register clock pclk_wdt: %ld\n", - __func__, PTR_ERR(clk)); - else - rockchip_clk_add_lookup(ctx, clk, PCLK_WDT); - rockchip_clk_register_plls(ctx, rk3288_pll_clks, ARRAY_SIZE(rk3288_pll_clks), RK3288_GRF_SOC_STATUS1); diff --git a/drivers/clk/rockchip/clk-rk3328.c b/drivers/clk/rockchip/clk-rk3328.c index 076b9777a955..c186a1985bf4 100644 --- a/drivers/clk/rockchip/clk-rk3328.c +++ b/drivers/clk/rockchip/clk-rk3328.c @@ -791,6 +791,9 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { GATE(PCLK_SARADC, "pclk_saradc", "pclk_bus", 0, RK3328_CLKGATE_CON(17), 15, GFLAGS), GATE(0, "pclk_pmu", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(28), 3, GFLAGS), + /* Watchdog pclk is controlled from the secure GRF */ + SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_bus"), + GATE(PCLK_USB3PHY_OTG, "pclk_usb3phy_otg", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(28), 1, GFLAGS), GATE(PCLK_USB3PHY_PIPE, "pclk_usb3phy_pipe", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(28), 2, GFLAGS), GATE(PCLK_USB3_GRF, "pclk_usb3_grf", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 2, GFLAGS), diff --git a/drivers/clk/rockchip/clk-rk3368.c b/drivers/clk/rockchip/clk-rk3368.c index 43b022d9393b..55443349439b 100644 --- a/drivers/clk/rockchip/clk-rk3368.c +++ b/drivers/clk/rockchip/clk-rk3368.c @@ -811,6 +811,9 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 2, GFLAGS), GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 1, GFLAGS), + /* Watchdog pclk is controlled by sgrf_soc_con3[7]. */ + SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_pd_alive"), + /* * pclk_vio gates * pclk_vio comes from the exactly same source as hclk_vio @@ -862,7 +865,6 @@ static void __init rk3368_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; - struct clk *clk; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -877,14 +879,6 @@ static void __init rk3368_clk_init(struct device_node *np) return; } - /* Watchdog pclk is controlled by sgrf_soc_con3[7]. */ - clk = clk_register_fixed_factor(NULL, "pclk_wdt", "pclk_pd_alive", 0, 1, 1); - if (IS_ERR(clk)) - pr_warn("%s: could not register clock pclk_wdt: %ld\n", - __func__, PTR_ERR(clk)); - else - rockchip_clk_add_lookup(ctx, clk, PCLK_WDT); - rockchip_clk_register_plls(ctx, rk3368_pll_clks, ARRAY_SIZE(rk3368_pll_clks), RK3368_GRF_SOC_STATUS0); diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c index a7ff71313278..ce1d2446f142 100644 --- a/drivers/clk/rockchip/clk-rk3399.c +++ b/drivers/clk/rockchip/clk-rk3399.c @@ -1295,6 +1295,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(PCLK_PMU_INTR_ARB, "pclk_pmu_intr_arb", "pclk_alive", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(31), 9, GFLAGS), GATE(PCLK_SGRF, "pclk_sgrf", "pclk_alive", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(31), 10, GFLAGS), + /* Watchdog pclk is controlled by RK3399 SECURE_GRF_SOC_CON3[8]. */ + SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_alive"), + GATE(SCLK_MIPIDPHY_REF, "clk_mipidphy_ref", "xin24m", 0, RK3399_CLKGATE_CON(11), 14, GFLAGS), GATE(SCLK_DPHY_PLL, "clk_dphy_pll", "clk_mipidphy_ref", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 0, GFLAGS), @@ -1522,7 +1525,6 @@ static void __init rk3399_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; - struct clk *clk; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -1537,14 +1539,6 @@ static void __init rk3399_clk_init(struct device_node *np) return; } - /* Watchdog pclk is controlled by RK3399 SECURE_GRF_SOC_CON3[8]. */ - clk = clk_register_fixed_factor(NULL, "pclk_wdt", "pclk_alive", 0, 1, 1); - if (IS_ERR(clk)) - pr_warn("%s: could not register clock pclk_wdt: %ld\n", - __func__, PTR_ERR(clk)); - else - rockchip_clk_add_lookup(ctx, clk, PCLK_WDT); - rockchip_clk_register_plls(ctx, rk3399_pll_clks, ARRAY_SIZE(rk3399_pll_clks), -1); diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h index adb66cc94929..b811597a3d38 100644 --- a/drivers/clk/rockchip/clk.h +++ b/drivers/clk/rockchip/clk.h @@ -811,6 +811,10 @@ struct rockchip_clk_branch { .gate_offset = -1, \ } +/* SGRF clocks are only accessible from secure mode, so not controllable */ +#define SGRF_GATE(_id, cname, pname) \ + FACTOR(_id, cname, pname, 0, 1, 1) + struct rockchip_clk_provider *rockchip_clk_init(struct device_node *np, void __iomem *base, unsigned long nr_clks); void rockchip_clk_of_add_provider(struct device_node *np, diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c index 982eb02bafda..51564fc23c63 100644 --- a/drivers/clk/samsung/clk-exynos4.c +++ b/drivers/clk/samsung/clk-exynos4.c @@ -958,6 +958,7 @@ static const struct samsung_gate_clock exynos4210_gate_clks[] __initconst = { /* list of gate clocks supported in exynos4x12 soc */ static const struct samsung_gate_clock exynos4x12_gate_clks[] __initconst = { + GATE(CLK_ASYNC_G3D, "async_g3d", "aclk200", GATE_IP_LEFTBUS, 6, 0, 0), GATE(CLK_AUDSS, "audss", "sclk_epll", E4X12_GATE_IP_MAUDIO, 0, 0, 0), GATE(CLK_MDNIE0, "mdnie0", "aclk160", GATE_IP_LCD0, 2, 0, 0), GATE(CLK_ROTATOR, "rotator", "aclk200", E4X12_GATE_IP_IMAGE, 1, 0, 0), diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c index 12d800fd9528..01bca5a498b2 100644 --- a/drivers/clk/samsung/clk-exynos5420.c +++ b/drivers/clk/samsung/clk-exynos5420.c @@ -131,6 +131,8 @@ #define SRC_CDREX 0x20200 #define DIV_CDREX0 0x20500 #define DIV_CDREX1 0x20504 +#define GATE_BUS_CDREX0 0x20700 +#define GATE_BUS_CDREX1 0x20704 #define KPLL_LOCK 0x28000 #define KPLL_CON0 0x28100 #define SRC_KFC 0x28200 @@ -245,6 +247,8 @@ static const unsigned long exynos5x_clk_regs[] __initconst = { DIV_CDREX1, SRC_KFC, DIV_KFC0, + GATE_BUS_CDREX0, + GATE_BUS_CDREX1, }; static const unsigned long exynos5800_clk_regs[] __initconst = { @@ -422,6 +426,9 @@ PNAME(mout_group13_5800_p) = { "dout_osc_div", "mout_sw_aclkfl1_550_cam" }; PNAME(mout_group14_5800_p) = { "dout_aclk550_cam", "dout_sclk_sw" }; PNAME(mout_group15_5800_p) = { "dout_osc_div", "mout_sw_aclk550_cam" }; PNAME(mout_group16_5800_p) = { "dout_osc_div", "mout_mau_epll_clk" }; +PNAME(mout_mx_mspll_ccore_phy_p) = { "sclk_bpll", "mout_sclk_dpll", + "mout_sclk_mpll", "ff_dout_spll2", + "mout_sclk_spll", "mout_sclk_epll"}; /* fixed rate clocks generated outside the soc */ static struct samsung_fixed_rate_clock @@ -447,7 +454,7 @@ static const struct samsung_fixed_factor_clock static const struct samsung_fixed_factor_clock exynos5800_fixed_factor_clks[] __initconst = { FFACTOR(0, "ff_dout_epll2", "mout_sclk_epll", 1, 2, 0), - FFACTOR(0, "ff_dout_spll2", "mout_sclk_spll", 1, 2, 0), + FFACTOR(CLK_FF_DOUT_SPLL2, "ff_dout_spll2", "mout_sclk_spll", 1, 2, 0), }; static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = { @@ -469,11 +476,14 @@ static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = { MUX(0, "mout_aclk300_disp1", mout_group5_5800_p, SRC_TOP2, 24, 2), MUX(0, "mout_aclk300_gscl", mout_group5_5800_p, SRC_TOP2, 28, 2), + MUX(CLK_MOUT_MX_MSPLL_CCORE_PHY, "mout_mx_mspll_ccore_phy", + mout_mx_mspll_ccore_phy_p, SRC_TOP7, 0, 3), + MUX(CLK_MOUT_MX_MSPLL_CCORE, "mout_mx_mspll_ccore", - mout_mx_mspll_ccore_p, SRC_TOP7, 16, 2), + mout_mx_mspll_ccore_p, SRC_TOP7, 16, 3), MUX_F(CLK_MOUT_MAU_EPLL, "mout_mau_epll_clk", mout_mau_epll_clk_5800_p, SRC_TOP7, 20, 2, CLK_SET_RATE_PARENT, 0), - MUX(0, "sclk_bpll", mout_bpll_p, SRC_TOP7, 24, 1), + MUX(CLK_SCLK_BPLL, "sclk_bpll", mout_bpll_p, SRC_TOP7, 24, 1), MUX(0, "mout_epll2", mout_epll2_5800_p, SRC_TOP7, 28, 1), MUX(0, "mout_aclk550_cam", mout_group3_5800_p, SRC_TOP8, 16, 3), @@ -645,7 +655,7 @@ static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = { MUX(0, "mout_sclk_mpll", mout_mpll_p, SRC_TOP6, 0, 1), MUX(CLK_MOUT_VPLL, "mout_sclk_vpll", mout_vpll_p, SRC_TOP6, 4, 1), - MUX(0, "mout_sclk_spll", mout_spll_p, SRC_TOP6, 8, 1), + MUX(CLK_MOUT_SCLK_SPLL, "mout_sclk_spll", mout_spll_p, SRC_TOP6, 8, 1), MUX(0, "mout_sclk_ipll", mout_ipll_p, SRC_TOP6, 12, 1), MUX(0, "mout_sclk_rpll", mout_rpll_p, SRC_TOP6, 16, 1), MUX_F(CLK_MOUT_EPLL, "mout_sclk_epll", mout_epll_p, SRC_TOP6, 20, 1, @@ -803,8 +813,21 @@ static const struct samsung_div_clock exynos5x_div_clks[] __initconst = { "mout_aclk400_disp1", DIV_TOP2, 4, 3), /* CDREX Block */ - DIV(CLK_DOUT_PCLK_CDREX, "dout_pclk_cdrex", "dout_aclk_cdrex1", - DIV_CDREX0, 28, 3), + /* + * The three clocks below are controlled using the same register and + * bits. They are put into one because there is a need of + * synchronization between the BUS and DREXs (two external memory + * interfaces). + * They are put here to show this HW assumption and for clock + * information summary completeness. + */ + DIV_F(CLK_DOUT_PCLK_CDREX, "dout_pclk_cdrex", "dout_aclk_cdrex1", + DIV_CDREX0, 28, 3, CLK_GET_RATE_NOCACHE, 0), + DIV_F(CLK_DOUT_PCLK_DREX0, "dout_pclk_drex0", "dout_cclk_drex0", + DIV_CDREX0, 28, 3, CLK_GET_RATE_NOCACHE, 0), + DIV_F(CLK_DOUT_PCLK_DREX1, "dout_pclk_drex1", "dout_cclk_drex0", + DIV_CDREX0, 28, 3, CLK_GET_RATE_NOCACHE, 0), + DIV_F(CLK_DOUT_SCLK_CDREX, "dout_sclk_cdrex", "mout_mclk_cdrex", DIV_CDREX0, 24, 3, CLK_SET_RATE_PARENT, 0), DIV(CLK_DOUT_ACLK_CDREX1, "dout_aclk_cdrex1", "dout_clk2x_phy0", @@ -1167,6 +1190,32 @@ static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = { GATE_TOP_SCLK_ISP, 12, CLK_SET_RATE_PARENT, 0), GATE(CLK_G3D, "g3d", "mout_user_aclk_g3d", GATE_IP_G3D, 9, 0, 0), + + /* CDREX */ + GATE(CLK_CLKM_PHY0, "clkm_phy0", "dout_sclk_cdrex", + GATE_BUS_CDREX0, 0, 0, 0), + GATE(CLK_CLKM_PHY1, "clkm_phy1", "dout_sclk_cdrex", + GATE_BUS_CDREX0, 1, 0, 0), + GATE(0, "mx_mspll_ccore_phy", "mout_mx_mspll_ccore_phy", + SRC_MASK_TOP7, 0, CLK_IGNORE_UNUSED, 0), + + GATE(CLK_ACLK_PPMU_DREX1_1, "aclk_ppmu_drex1_1", "dout_aclk_cdrex1", + GATE_BUS_CDREX1, 12, CLK_IGNORE_UNUSED, 0), + GATE(CLK_ACLK_PPMU_DREX1_0, "aclk_ppmu_drex1_0", "dout_aclk_cdrex1", + GATE_BUS_CDREX1, 13, CLK_IGNORE_UNUSED, 0), + GATE(CLK_ACLK_PPMU_DREX0_1, "aclk_ppmu_drex0_1", "dout_aclk_cdrex1", + GATE_BUS_CDREX1, 14, CLK_IGNORE_UNUSED, 0), + GATE(CLK_ACLK_PPMU_DREX0_0, "aclk_ppmu_drex0_0", "dout_aclk_cdrex1", + GATE_BUS_CDREX1, 15, CLK_IGNORE_UNUSED, 0), + + GATE(CLK_PCLK_PPMU_DREX1_1, "pclk_ppmu_drex1_1", "dout_pclk_cdrex", + GATE_BUS_CDREX1, 26, CLK_IGNORE_UNUSED, 0), + GATE(CLK_PCLK_PPMU_DREX1_0, "pclk_ppmu_drex1_0", "dout_pclk_cdrex", + GATE_BUS_CDREX1, 27, CLK_IGNORE_UNUSED, 0), + GATE(CLK_PCLK_PPMU_DREX0_1, "pclk_ppmu_drex0_1", "dout_pclk_cdrex", + GATE_BUS_CDREX1, 28, CLK_IGNORE_UNUSED, 0), + GATE(CLK_PCLK_PPMU_DREX0_0, "pclk_ppmu_drex0_0", "dout_pclk_cdrex", + GATE_BUS_CDREX1, 29, CLK_IGNORE_UNUSED, 0), }; static const struct samsung_div_clock exynos5x_disp_div_clks[] __initconst = { @@ -1282,6 +1331,17 @@ static const struct samsung_pll_rate_table exynos5420_pll2550x_24mhz_tbl[] __ini PLL_35XX_RATE(24 * MHZ, 200000000, 200, 3, 3), }; +static const struct samsung_pll_rate_table exynos5422_bpll_rate_table[] = { + PLL_35XX_RATE(24 * MHZ, 825000000, 275, 4, 1), + PLL_35XX_RATE(24 * MHZ, 728000000, 182, 3, 1), + PLL_35XX_RATE(24 * MHZ, 633000000, 211, 4, 1), + PLL_35XX_RATE(24 * MHZ, 543000000, 181, 2, 2), + PLL_35XX_RATE(24 * MHZ, 413000000, 413, 6, 2), + PLL_35XX_RATE(24 * MHZ, 275000000, 275, 3, 3), + PLL_35XX_RATE(24 * MHZ, 206000000, 206, 3, 3), + PLL_35XX_RATE(24 * MHZ, 165000000, 110, 2, 3), +}; + static const struct samsung_pll_rate_table exynos5420_epll_24mhz_tbl[] = { PLL_36XX_RATE(24 * MHZ, 600000000U, 100, 2, 1, 0), PLL_36XX_RATE(24 * MHZ, 400000000U, 200, 3, 2, 0), @@ -1424,9 +1484,13 @@ static void __init exynos5x_clk_init(struct device_node *np, exynos5x_plls[apll].rate_table = exynos5420_pll2550x_24mhz_tbl; exynos5x_plls[epll].rate_table = exynos5420_epll_24mhz_tbl; exynos5x_plls[kpll].rate_table = exynos5420_pll2550x_24mhz_tbl; - exynos5x_plls[bpll].rate_table = exynos5420_pll2550x_24mhz_tbl; } + if (soc == EXYNOS5420) + exynos5x_plls[bpll].rate_table = exynos5420_pll2550x_24mhz_tbl; + else + exynos5x_plls[bpll].rate_table = exynos5422_bpll_rate_table; + samsung_clk_register_pll(ctx, exynos5x_plls, ARRAY_SIZE(exynos5x_plls), reg_base); samsung_clk_register_fixed_rate(ctx, exynos5x_fixed_rate_clks, diff --git a/drivers/clk/samsung/clk-exynos5433.c b/drivers/clk/samsung/clk-exynos5433.c index 945d5f2ad733..7824c2ba3d8e 100644 --- a/drivers/clk/samsung/clk-exynos5433.c +++ b/drivers/clk/samsung/clk-exynos5433.c @@ -5587,8 +5587,8 @@ static int __init exynos5433_cmu_probe(struct platform_device *pdev) data->nr_clk_save = info->nr_clk_regs; data->clk_suspend = info->suspend_regs; data->nr_clk_suspend = info->nr_suspend_regs; - data->nr_pclks = of_count_phandle_with_args(dev->of_node, "clocks", - "#clock-cells"); + data->nr_pclks = of_clk_get_parent_count(dev->of_node); + if (data->nr_pclks > 0) { data->pclks = devm_kcalloc(dev, sizeof(struct clk *), data->nr_pclks, GFP_KERNEL); diff --git a/drivers/clk/socfpga/clk-s10.c b/drivers/clk/socfpga/clk-s10.c index 5bed36e12951..993f3a73c71e 100644 --- a/drivers/clk/socfpga/clk-s10.c +++ b/drivers/clk/socfpga/clk-s10.c @@ -161,8 +161,12 @@ static const struct stratix10_gate_clock s10_gate_clks[] = { 8, 0, 0, 0, 0, 0, 0}, { STRATIX10_SPI_M_CLK, "spi_m_clk", "l4_mp_clk", NULL, 1, 0, 0xA4, 9, 0, 0, 0, 0, 0, 0}, - { STRATIX10_NAND_CLK, "nand_clk", "l4_main_clk", NULL, 1, 0, 0xA4, + { STRATIX10_NAND_X_CLK, "nand_x_clk", "l4_mp_clk", NULL, 1, 0, 0xA4, 10, 0, 0, 0, 0, 0, 0}, + { STRATIX10_NAND_CLK, "nand_clk", "nand_x_clk", NULL, 1, 0, 0xA4, + 10, 0, 0, 0, 0, 0, 4}, + { STRATIX10_NAND_ECC_CLK, "nand_ecc_clk", "nand_x_clk", NULL, 1, 0, 0xA4, + 10, 0, 0, 0, 0, 0, 4}, }; static int s10_clk_register_c_perip(const struct stratix10_perip_c_clock *clks, diff --git a/drivers/clk/sprd/common.c b/drivers/clk/sprd/common.c index e038b0447206..a5bdca1de5d0 100644 --- a/drivers/clk/sprd/common.c +++ b/drivers/clk/sprd/common.c @@ -42,6 +42,7 @@ int sprd_clk_regmap_init(struct platform_device *pdev, void __iomem *base; struct device_node *node = pdev->dev.of_node; struct regmap *regmap; + struct resource *res; if (of_find_property(node, "sprd,syscon", NULL)) { regmap = syscon_regmap_lookup_by_phandle(node, "sprd,syscon"); @@ -50,10 +51,14 @@ int sprd_clk_regmap_init(struct platform_device *pdev, return PTR_ERR(regmap); } } else { - base = of_iomap(node, 0); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + regmap = devm_regmap_init_mmio(&pdev->dev, base, &sprdclk_regmap_config); - if (IS_ERR_OR_NULL(regmap)) { + if (IS_ERR(regmap)) { pr_err("failed to init regmap\n"); return PTR_ERR(regmap); } diff --git a/drivers/clk/sprd/sc9860-clk.c b/drivers/clk/sprd/sc9860-clk.c index 9980ab55271b..f76305b4bc8d 100644 --- a/drivers/clk/sprd/sc9860-clk.c +++ b/drivers/clk/sprd/sc9860-clk.c @@ -2023,6 +2023,7 @@ static int sc9860_clk_probe(struct platform_device *pdev) { const struct of_device_id *match; const struct sprd_clk_desc *desc; + int ret; match = of_match_node(sprd_sc9860_clk_ids, pdev->dev.of_node); if (!match) { @@ -2031,7 +2032,9 @@ static int sc9860_clk_probe(struct platform_device *pdev) } desc = match->data; - sprd_clk_regmap_init(pdev, desc); + ret = sprd_clk_regmap_init(pdev, desc); + if (ret) + return ret; return sprd_clk_probe(&pdev->dev, desc->hw_clks); } diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c index df43952e403e..f32366d9336e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c +++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c @@ -160,8 +160,9 @@ static struct ccu_nk pll_periph_base_clk = { }, }; -static CLK_FIXED_FACTOR(pll_periph_clk, "pll-periph", "pll-periph-base", - 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph_clk, "pll-periph", + &pll_periph_base_clk.common.hw, + 2, 1, CLK_SET_RATE_PARENT); /* Not documented on A10 */ static struct ccu_div pll_periph_sata_clk = { @@ -1028,19 +1029,29 @@ static struct ccu_common *sun4i_sun7i_ccu_clks[] = { &out_b_clk.common }; +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + /* Post-divider for pll-audio is hardcoded to 1 */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x", - "pll-video0", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x", - "pll-video1", 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x", + &pll_video0_clk.common.hw, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x", + &pll_video1_clk.common.hw, + 1, 2, CLK_SET_RATE_PARENT); static struct clk_hw_onecell_data sun4i_a10_hw_clks = { diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c index 1786ee8fe8bb..49bd7a4c015c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c @@ -597,23 +597,34 @@ static SUNXI_CCU_M_WITH_GATE(gpu_clk, "gpu", "pll-gpu", 0x1a0, 0, 3, BIT(31), CLK_SET_RATE_PARENT); /* Fixed Factor clocks */ -static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 2, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0); + +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; /* We hardcode the divider to 1 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x", - "pll-periph0", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x", - "pll-periph1", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x", - "pll-video0", 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x", + &pll_periph0_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_periph1_2x_clk, "pll-periph1-2x", + &pll_periph1_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x", + &pll_video0_clk.common.hw, + 1, 2, CLK_SET_RATE_PARENT); static struct ccu_common *sun50i_a64_ccu_clks[] = { &pll_cpux_clk.common, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c index 27554eaf6929..45a1ed3fe674 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c @@ -49,7 +49,7 @@ static struct ccu_div ar100_clk = { }, }; -static CLK_FIXED_FACTOR(r_ahb_clk, "r-ahb", "ar100", 1, 1, 0); +static CLK_FIXED_FACTOR_HW(r_ahb_clk, "r-ahb", &ar100_clk.common.hw, 1, 1, 0); static struct ccu_div r_apb1_clk = { .div = _SUNXI_CCU_DIV(0, 2), @@ -104,7 +104,7 @@ static SUNXI_CCU_GATE(r_apb2_i2c_clk, "r-apb2-i2c", "r-apb2", static SUNXI_CCU_GATE(r_apb1_ir_clk, "r-apb1-ir", "r-apb1", 0x1cc, BIT(0), 0); static SUNXI_CCU_GATE(r_apb1_w1_clk, "r-apb1-w1", "r-apb1", - 0x1cc, BIT(0), 0); + 0x1ec, BIT(0), 0); /* Information of IR(RX) mod clock is gathered from BSP source code */ static const char * const r_mod0_default_parents[] = { "osc32k", "osc24M" }; diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c index 9d3f98962779..aebef4af9861 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c @@ -622,8 +622,9 @@ static SUNXI_CCU_GATE(bus_xhci_clk, "bus-xhci", "ahb3", 0xa8c, BIT(5), 0); static SUNXI_CCU_GATE(bus_ehci3_clk, "bus-ehci3", "ahb3", 0xa8c, BIT(7), 0); static SUNXI_CCU_GATE(bus_otg_clk, "bus-otg", "ahb3", 0xa8c, BIT(8), 0); -static CLK_FIXED_FACTOR(pcie_ref_100m_clk, "pcie-ref-100M", - "pll-periph0-4x", 24, 1, 0); +static struct clk_fixed_factor pll_periph0_4x_clk; +static CLK_FIXED_FACTOR_HW(pcie_ref_100m_clk, "pcie-ref-100M", + &pll_periph0_4x_clk.hw, 24, 1, 0); static SUNXI_CCU_GATE(pcie_ref_clk, "pcie-ref", "pcie-ref-100M", 0xab0, BIT(31), 0); static SUNXI_CCU_GATE(pcie_ref_out_clk, "pcie-ref-out", "pcie-ref", @@ -745,34 +746,52 @@ static SUNXI_CCU_M_WITH_MUX_GATE(hdcp_clk, "hdcp", hdcp_parents, 0xc40, static SUNXI_CCU_GATE(bus_hdcp_clk, "bus-hdcp", "ahb3", 0xc4c, BIT(0), 0); /* Fixed factor clocks */ -static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 2, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0); + +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; /* * The divider of pll-audio is fixed to 8 now, as pll-audio-4x has a * fixed post-divider 2. */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 8, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); - -static CLK_FIXED_FACTOR(pll_periph0_4x_clk, "pll-periph0-4x", - "pll-periph0", 1, 4, 0); -static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x", - "pll-periph0", 1, 2, 0); - -static CLK_FIXED_FACTOR(pll_periph1_4x_clk, "pll-periph1-4x", - "pll-periph1", 1, 4, 0); -static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x", - "pll-periph1", 1, 2, 0); - -static CLK_FIXED_FACTOR(pll_video0_4x_clk, "pll-video0-4x", - "pll-video0", 1, 4, CLK_SET_RATE_PARENT); - -static CLK_FIXED_FACTOR(pll_video1_4x_clk, "pll-video1-4x", - "pll-video1", 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 8, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 4, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); + +static const struct clk_hw *pll_periph0_parents[] = { + &pll_periph0_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_periph0_4x_clk, "pll-periph0-4x", + pll_periph0_parents, + 1, 4, 0); +static CLK_FIXED_FACTOR_HWS(pll_periph0_2x_clk, "pll-periph0-2x", + pll_periph0_parents, + 1, 2, 0); + +static const struct clk_hw *pll_periph1_parents[] = { + &pll_periph1_clk.common.hw +}; +static CLK_FIXED_FACTOR_HWS(pll_periph1_4x_clk, "pll-periph1-4x", + pll_periph1_parents, + 1, 4, 0); +static CLK_FIXED_FACTOR_HWS(pll_periph1_2x_clk, "pll-periph1-2x", + pll_periph1_parents, + 1, 2, 0); + +static CLK_FIXED_FACTOR_HW(pll_video0_4x_clk, "pll-video0-4x", + &pll_video0_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video1_4x_clk, "pll-video1-4x", + &pll_video1_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); static struct ccu_common *sun50i_h6_ccu_clks[] = { &pll_cpux_clk.common, diff --git a/drivers/clk/sunxi-ng/ccu-sun5i.c b/drivers/clk/sunxi-ng/ccu-sun5i.c index b71ed0f6f785..b78e9b507c1c 100644 --- a/drivers/clk/sunxi-ng/ccu-sun5i.c +++ b/drivers/clk/sunxi-ng/ccu-sun5i.c @@ -603,19 +603,29 @@ static struct ccu_common *sun5i_a10s_ccu_clks[] = { &iep_clk.common, }; +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + /* We hardcode the divider to 1 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x", - "pll-video0", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x", - "pll-video1", 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x", + &pll_video0_clk.common.hw, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x", + &pll_video1_clk.common.hw, + 1, 2, CLK_SET_RATE_PARENT); static struct clk_hw_onecell_data sun5i_a10s_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c index 2ff7b082df28..9b40d53266a3 100644 --- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c +++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c @@ -955,21 +955,32 @@ static struct ccu_common *sun6i_a31_ccu_clks[] = { &out_c_clk.common, }; +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + /* We hardcode the divider to 1 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x", - "pll-periph", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x", - "pll-video0", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x", - "pll-video1", 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph_2x_clk, "pll-periph-2x", + &pll_periph_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x", + &pll_video0_clk.common.hw, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x", + &pll_video1_clk.common.hw, + 1, 2, CLK_SET_RATE_PARENT); static struct clk_hw_onecell_data sun6i_a31_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c index 14ced502788a..103aa504f6c8 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c @@ -543,19 +543,29 @@ static struct ccu_common *sun8i_a23_ccu_clks[] = { &ats_clk.common, }; +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + /* We hardcode the divider to 1 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x", - "pll-periph", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x", - "pll-video", 1, 2, 0); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph_2x_clk, "pll-periph-2x", + &pll_periph_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_video_2x_clk, "pll-video-2x", + &pll_video_clk.common.hw, + 1, 2, 0); static struct clk_hw_onecell_data sun8i_a23_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c index 61fb41f4903c..91838cd11037 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c @@ -580,19 +580,29 @@ static struct ccu_common *sun8i_a33_ccu_clks[] = { &ats_clk.common, }; +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + /* We hardcode the divider to 1 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x", - "pll-periph", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x", - "pll-video", 1, 2, 0); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph_2x_clk, "pll-periph-2x", + &pll_periph_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_video_2x_clk, "pll-video-2x", + &pll_video_clk.common.hw, + 1, 2, 0); static struct clk_hw_onecell_data sun8i_a33_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c index 9601504905b2..6b636362379e 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c @@ -717,17 +717,26 @@ static struct ccu_common *sun50i_h5_ccu_clks[] = { &gpu_clk.common, }; +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + /* We hardcode the divider to 1 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x", - "pll-periph0", 1, 2, 0); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x", + &pll_periph0_clk.common.hw, + 1, 2, 0); static struct clk_hw_onecell_data sun8i_h3_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.c b/drivers/clk/sunxi-ng/ccu-sun8i-r.c index b5be11e5de0d..4646fdc61053 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.c @@ -17,10 +17,13 @@ #include "ccu-sun8i-r.h" -static const char * const ar100_parents[] = { "osc32k", "osc24M", - "pll-periph0", "iosc" }; -static const char * const a83t_ar100_parents[] = { "osc16M-d512", "osc24M", - "pll-periph0", "iosc" }; +static const struct clk_parent_data ar100_parents[] = { + { .fw_name = "losc" }, + { .fw_name = "hosc" }, + { .fw_name = "pll-periph" }, + { .fw_name = "iosc" }, +}; + static const struct ccu_mux_var_prediv ar100_predivs[] = { { .index = 2, .shift = 8, .width = 5 }, }; @@ -39,64 +42,49 @@ static struct ccu_div ar100_clk = { .common = { .reg = 0x00, .features = CCU_FEATURE_VARIABLE_PREDIV, - .hw.init = CLK_HW_INIT_PARENTS("ar100", - ar100_parents, - &ccu_div_ops, - 0), - }, -}; - -static struct ccu_div a83t_ar100_clk = { - .div = _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO), - - .mux = { - .shift = 16, - .width = 2, - - .var_predivs = ar100_predivs, - .n_var_predivs = ARRAY_SIZE(ar100_predivs), - }, - - .common = { - .reg = 0x00, - .features = CCU_FEATURE_VARIABLE_PREDIV, - .hw.init = CLK_HW_INIT_PARENTS("ar100", - a83t_ar100_parents, - &ccu_div_ops, - 0), + .hw.init = CLK_HW_INIT_PARENTS_DATA("ar100", + ar100_parents, + &ccu_div_ops, + 0), }, }; -static CLK_FIXED_FACTOR(ahb0_clk, "ahb0", "ar100", 1, 1, 0); +static CLK_FIXED_FACTOR_HW(ahb0_clk, "ahb0", &ar100_clk.common.hw, 1, 1, 0); static struct ccu_div apb0_clk = { .div = _SUNXI_CCU_DIV_FLAGS(0, 2, CLK_DIVIDER_POWER_OF_TWO), .common = { .reg = 0x0c, - .hw.init = CLK_HW_INIT("apb0", - "ahb0", - &ccu_div_ops, - 0), + .hw.init = CLK_HW_INIT_HW("apb0", + &ahb0_clk.hw, + &ccu_div_ops, + 0), }, }; static SUNXI_CCU_M(a83t_apb0_clk, "apb0", "ahb0", 0x0c, 0, 2, 0); -static SUNXI_CCU_GATE(apb0_pio_clk, "apb0-pio", "apb0", - 0x28, BIT(0), 0); -static SUNXI_CCU_GATE(apb0_ir_clk, "apb0-ir", "apb0", - 0x28, BIT(1), 0); -static SUNXI_CCU_GATE(apb0_timer_clk, "apb0-timer", "apb0", - 0x28, BIT(2), 0); -static SUNXI_CCU_GATE(apb0_rsb_clk, "apb0-rsb", "apb0", - 0x28, BIT(3), 0); -static SUNXI_CCU_GATE(apb0_uart_clk, "apb0-uart", "apb0", - 0x28, BIT(4), 0); -static SUNXI_CCU_GATE(apb0_i2c_clk, "apb0-i2c", "apb0", - 0x28, BIT(6), 0); -static SUNXI_CCU_GATE(apb0_twd_clk, "apb0-twd", "apb0", - 0x28, BIT(7), 0); +/* + * Define the parent as an array that can be reused to save space + * instead of having compound literals for each gate. Also have it + * non-const so we can change it on the A83T. + */ +static const struct clk_hw *apb0_gate_parent[] = { &apb0_clk.common.hw }; +static SUNXI_CCU_GATE_HWS(apb0_pio_clk, "apb0-pio", + apb0_gate_parent, 0x28, BIT(0), 0); +static SUNXI_CCU_GATE_HWS(apb0_ir_clk, "apb0-ir", + apb0_gate_parent, 0x28, BIT(1), 0); +static SUNXI_CCU_GATE_HWS(apb0_timer_clk, "apb0-timer", + apb0_gate_parent, 0x28, BIT(2), 0); +static SUNXI_CCU_GATE_HWS(apb0_rsb_clk, "apb0-rsb", + apb0_gate_parent, 0x28, BIT(3), 0); +static SUNXI_CCU_GATE_HWS(apb0_uart_clk, "apb0-uart", + apb0_gate_parent, 0x28, BIT(4), 0); +static SUNXI_CCU_GATE_HWS(apb0_i2c_clk, "apb0-i2c", + apb0_gate_parent, 0x28, BIT(6), 0); +static SUNXI_CCU_GATE_HWS(apb0_twd_clk, "apb0-twd", + apb0_gate_parent, 0x28, BIT(7), 0); static const char * const r_mod0_default_parents[] = { "osc32k", "osc24M" }; static SUNXI_CCU_MP_WITH_MUX_GATE(ir_clk, "ir", @@ -107,7 +95,10 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(ir_clk, "ir", BIT(31), /* gate */ 0); -static const char *const a83t_r_mod0_parents[] = { "osc16M", "osc24M" }; +static const struct clk_parent_data a83t_r_mod0_parents[] = { + { .fw_name = "iosc" }, + { .fw_name = "hosc" }, +}; static const struct ccu_mux_fixed_prediv a83t_ir_predivs[] = { { .index = 0, .div = 16 }, }; @@ -127,15 +118,15 @@ static struct ccu_mp a83t_ir_clk = { .common = { .reg = 0x54, .features = CCU_FEATURE_VARIABLE_PREDIV, - .hw.init = CLK_HW_INIT_PARENTS("ir", - a83t_r_mod0_parents, - &ccu_mp_ops, - 0), + .hw.init = CLK_HW_INIT_PARENTS_DATA("ir", + a83t_r_mod0_parents, + &ccu_mp_ops, + 0), }, }; static struct ccu_common *sun8i_a83t_r_ccu_clks[] = { - &a83t_ar100_clk.common, + &ar100_clk.common, &a83t_apb0_clk.common, &apb0_pio_clk.common, &apb0_ir_clk.common, @@ -174,7 +165,7 @@ static struct ccu_common *sun50i_a64_r_ccu_clks[] = { static struct clk_hw_onecell_data sun8i_a83t_r_hw_clks = { .hws = { - [CLK_AR100] = &a83t_ar100_clk.common.hw, + [CLK_AR100] = &ar100_clk.common.hw, [CLK_AHB0] = &ahb0_clk.hw, [CLK_APB0] = &a83t_apb0_clk.common.hw, [CLK_APB0_PIO] = &apb0_pio_clk.common.hw, @@ -291,6 +282,9 @@ static void __init sunxi_r_ccu_init(struct device_node *node, static void __init sun8i_a83t_r_ccu_setup(struct device_node *node) { + /* Fix apb0 bus gate parents here */ + apb0_gate_parent[0] = &a83t_apb0_clk.common.hw; + sunxi_r_ccu_init(node, &sun8i_a83t_r_ccu_desc); } CLK_OF_DECLARE(sun8i_a83t_r_ccu, "allwinner,sun8i-a83t-r-ccu", diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c index 540f5f7454fc..897490800102 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c @@ -944,25 +944,37 @@ static struct ccu_common *sun8i_r40_ccu_clks[] = { }; /* Fixed Factor clocks */ -static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 2, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0); + +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; /* We hardcode the divider to 4 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x", - "pll-periph0", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x", - "pll-periph1", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x", - "pll-video0", 1, 2, 0); -static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x", - "pll-video1", 1, 2, 0); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 4, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x", + &pll_periph0_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_periph1_2x_clk, "pll-periph1-2x", + &pll_periph1_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_video0_2x_clk, "pll-video0-2x", + &pll_video0_clk.common.hw, + 1, 2, 0); +static CLK_FIXED_FACTOR_HW(pll_video1_2x_clk, "pll-video1-2x", + &pll_video1_clk.common.hw, + 1, 2, 0); static struct clk_hw_onecell_data sun8i_r40_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c index cbbf06d42c2c..9b3939fc7faa 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c @@ -429,17 +429,26 @@ static struct ccu_common *sun8i_v3s_ccu_clks[] = { &mipi_csi_clk.common, }; +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + /* We hardcode the divider to 4 for now */ -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x", - "pll-periph0", 1, 2, 0); +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 4, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_periph0_2x_clk, "pll-periph0-2x", + &pll_periph0_clk.common.hw, + 1, 2, 0); static struct clk_hw_onecell_data sun8i_v3s_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c index 2f82cd855b0f..4b4a507d04ed 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c @@ -14,18 +14,26 @@ #include "ccu-sun9i-a80-usb.h" -static SUNXI_CCU_GATE(bus_hci0_clk, "bus-hci0", "bus-usb", 0x0, BIT(1), 0); -static SUNXI_CCU_GATE(usb_ohci0_clk, "usb-ohci0", "osc24M", 0x0, BIT(2), 0); -static SUNXI_CCU_GATE(bus_hci1_clk, "bus-hci1", "bus-usb", 0x0, BIT(3), 0); -static SUNXI_CCU_GATE(bus_hci2_clk, "bus-hci2", "bus-usb", 0x0, BIT(5), 0); -static SUNXI_CCU_GATE(usb_ohci2_clk, "usb-ohci2", "osc24M", 0x0, BIT(6), 0); - -static SUNXI_CCU_GATE(usb0_phy_clk, "usb0-phy", "osc24M", 0x4, BIT(1), 0); -static SUNXI_CCU_GATE(usb1_hsic_clk, "usb1-hsic", "osc24M", 0x4, BIT(2), 0); -static SUNXI_CCU_GATE(usb1_phy_clk, "usb1-phy", "osc24M", 0x4, BIT(3), 0); -static SUNXI_CCU_GATE(usb2_hsic_clk, "usb2-hsic", "osc24M", 0x4, BIT(4), 0); -static SUNXI_CCU_GATE(usb2_phy_clk, "usb2-phy", "osc24M", 0x4, BIT(5), 0); -static SUNXI_CCU_GATE(usb_hsic_clk, "usb-hsic", "osc24M", 0x4, BIT(10), 0); +static const struct clk_parent_data clk_parent_hosc[] = { + { .fw_name = "hosc" }, +}; + +static const struct clk_parent_data clk_parent_bus[] = { + { .fw_name = "bus" }, +}; + +static SUNXI_CCU_GATE_DATA(bus_hci0_clk, "bus-hci0", clk_parent_bus, 0x0, BIT(1), 0); +static SUNXI_CCU_GATE_DATA(usb_ohci0_clk, "usb-ohci0", clk_parent_hosc, 0x0, BIT(2), 0); +static SUNXI_CCU_GATE_DATA(bus_hci1_clk, "bus-hci1", clk_parent_bus, 0x0, BIT(3), 0); +static SUNXI_CCU_GATE_DATA(bus_hci2_clk, "bus-hci2", clk_parent_bus, 0x0, BIT(5), 0); +static SUNXI_CCU_GATE_DATA(usb_ohci2_clk, "usb-ohci2", clk_parent_hosc, 0x0, BIT(6), 0); + +static SUNXI_CCU_GATE_DATA(usb0_phy_clk, "usb0-phy", clk_parent_hosc, 0x4, BIT(1), 0); +static SUNXI_CCU_GATE_DATA(usb1_hsic_clk, "usb1-hsic", clk_parent_hosc, 0x4, BIT(2), 0); +static SUNXI_CCU_GATE_DATA(usb1_phy_clk, "usb1-phy", clk_parent_hosc, 0x4, BIT(3), 0); +static SUNXI_CCU_GATE_DATA(usb2_hsic_clk, "usb2-hsic", clk_parent_hosc, 0x4, BIT(4), 0); +static SUNXI_CCU_GATE_DATA(usb2_phy_clk, "usb2-phy", clk_parent_hosc, 0x4, BIT(5), 0); +static SUNXI_CCU_GATE_DATA(usb_hsic_clk, "usb-hsic", clk_parent_hosc, 0x4, BIT(10), 0); static struct ccu_common *sun9i_a80_usb_clks[] = { &bus_hci0_clk.common, diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c index e748b8a6f3c5..7ecc3a5a5b5e 100644 --- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c +++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c @@ -374,16 +374,25 @@ static struct ccu_common *suniv_ccu_clks[] = { &avs_clk.common, }; -static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", - "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", - "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", - "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", - "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); -static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x", - "pll-video", 1, 2, 0); +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_base_clk.common.hw +}; + +static CLK_FIXED_FACTOR_HWS(pll_audio_clk, "pll-audio", + clk_parent_pll_audio, + 4, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_8x_clk, "pll-audio-8x", + clk_parent_pll_audio, + 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video_2x_clk, "pll-video-2x", + &pll_video_clk.common.hw, + 1, 2, 0); static struct clk_hw_onecell_data suniv_hw_clks = { .hws = { diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c index c173778c8a78..7fe3ac980e5f 100644 --- a/drivers/clk/sunxi-ng/ccu_common.c +++ b/drivers/clk/sunxi-ng/ccu_common.c @@ -101,7 +101,7 @@ int sunxi_ccu_probe(struct device_node *node, void __iomem *reg, if (!hw) continue; - ret = clk_hw_register(NULL, hw); + ret = of_clk_hw_register(node, hw); if (ret) { pr_err("Couldn't register clock %d - %s\n", i, clk_hw_get_name(hw)); diff --git a/drivers/clk/sunxi-ng/ccu_gate.h b/drivers/clk/sunxi-ng/ccu_gate.h index da8100e8846d..c386689a952b 100644 --- a/drivers/clk/sunxi-ng/ccu_gate.h +++ b/drivers/clk/sunxi-ng/ccu_gate.h @@ -28,6 +28,59 @@ struct ccu_gate { } \ } +#define SUNXI_CCU_GATE_HW(_struct, _name, _parent, _reg, _gate, _flags) \ + struct ccu_gate _struct = { \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_HW(_name, \ + _parent, \ + &ccu_gate_ops, \ + _flags), \ + } \ + } + +#define SUNXI_CCU_GATE_FW(_struct, _name, _parent, _reg, _gate, _flags) \ + struct ccu_gate _struct = { \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_FW_NAME(_name, \ + _parent, \ + &ccu_gate_ops, \ + _flags), \ + } \ + } + +/* + * The following two macros allow the re-use of the data structure + * holding the parent info. + */ +#define SUNXI_CCU_GATE_HWS(_struct, _name, _parent, _reg, _gate, _flags) \ + struct ccu_gate _struct = { \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_HWS(_name, \ + _parent, \ + &ccu_gate_ops, \ + _flags), \ + } \ + } + +#define SUNXI_CCU_GATE_DATA(_struct, _name, _data, _reg, _gate, _flags) \ + struct ccu_gate _struct = { \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .hw.init = \ + CLK_HW_INIT_PARENTS_DATA(_name, \ + _data, \ + &ccu_gate_ops, \ + _flags), \ + } \ + } + static inline struct ccu_gate *hw_to_ccu_gate(struct clk_hw *hw) { struct ccu_common *common = hw_to_ccu_common(hw); diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c index 623fda5e911f..d3a43381a792 100644 --- a/drivers/clk/sunxi/clk-sunxi.c +++ b/drivers/clk/sunxi/clk-sunxi.c @@ -980,6 +980,8 @@ static struct clk ** __init sunxi_divs_clk_setup(struct device_node *node, if (endp) { derived_name = kstrndup(clk_name, endp - clk_name, GFP_KERNEL); + if (!derived_name) + return NULL; factors.name = derived_name; } else { factors.name = clk_name; diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c index ac1d27a8c650..df172d5772d7 100644 --- a/drivers/clk/tegra/clk-tegra210.c +++ b/drivers/clk/tegra/clk-tegra210.c @@ -984,8 +984,6 @@ static void tegra210_pllre_set_defaults(struct tegra_clk_pll *pllre) pllre->params->defaults_set = true; if (val & PLL_ENABLE) { - pr_warn("PLL_RE already enabled. Postponing set full defaults\n"); - /* * PLL is ON: check if defaults already set, then set those * that can be updated in flight. @@ -1005,13 +1003,20 @@ static void tegra210_pllre_set_defaults(struct tegra_clk_pll *pllre) _pll_misc_chk_default(clk_base, pllre->params, 0, val, ~mask & PLLRE_MISC0_WRITE_MASK); - /* Enable lock detect */ + /* The PLL doesn't work if it's in IDDQ. */ val = readl_relaxed(clk_base + pllre->params->ext_misc_reg[0]); + if (val & PLLRE_MISC0_IDDQ) + pr_warn("unexpected IDDQ bit set for enabled clock\n"); + + /* Enable lock detect */ val &= ~mask; val |= PLLRE_MISC0_DEFAULT_VALUE & mask; writel_relaxed(val, clk_base + pllre->params->ext_misc_reg[0]); udelay(1); + if (!pllre->params->defaults_set) + pr_warn("PLL_RE already enabled. Postponing set full defaults\n"); + return; } @@ -2204,9 +2209,9 @@ static struct div_nmp pllu_nmp = { }; static struct tegra_clk_pll_freq_table pll_u_freq_table[] = { - { 12000000, 480000000, 40, 1, 0, 0 }, - { 13000000, 480000000, 36, 1, 0, 0 }, /* actual: 468.0 MHz */ - { 38400000, 480000000, 25, 2, 0, 0 }, + { 12000000, 480000000, 40, 1, 1, 0 }, + { 13000000, 480000000, 36, 1, 1, 0 }, /* actual: 468.0 MHz */ + { 38400000, 480000000, 25, 2, 1, 0 }, { 0, 0, 0, 0, 0, 0 }, }; @@ -3332,7 +3337,7 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA210_CLK_DFLL_SOC, TEGRA210_CLK_PLL_P, 51000000, 1 }, { TEGRA210_CLK_DFLL_REF, TEGRA210_CLK_PLL_P, 51000000, 1 }, { TEGRA210_CLK_SBC4, TEGRA210_CLK_PLL_P, 12000000, 1 }, - { TEGRA210_CLK_PLL_RE_VCO, TEGRA210_CLK_CLK_MAX, 672000000, 1 }, + { TEGRA210_CLK_PLL_U_OUT1, TEGRA210_CLK_CLK_MAX, 48000000, 1 }, { TEGRA210_CLK_XUSB_GATE, TEGRA210_CLK_CLK_MAX, 0, 1 }, { TEGRA210_CLK_XUSB_SS_SRC, TEGRA210_CLK_PLL_U_480M, 120000000, 0 }, { TEGRA210_CLK_XUSB_FS_SRC, TEGRA210_CLK_PLL_U_48M, 48000000, 0 }, @@ -3357,7 +3362,6 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA210_CLK_PLL_DP, TEGRA210_CLK_CLK_MAX, 270000000, 0 }, { TEGRA210_CLK_SOC_THERM, TEGRA210_CLK_PLL_P, 51000000, 0 }, { TEGRA210_CLK_CCLK_G, TEGRA210_CLK_CLK_MAX, 0, 1 }, - { TEGRA210_CLK_PLL_U_OUT1, TEGRA210_CLK_CLK_MAX, 48000000, 1 }, { TEGRA210_CLK_PLL_U_OUT2, TEGRA210_CLK_CLK_MAX, 60000000, 1 }, { TEGRA210_CLK_SPDIF_IN_SYNC, TEGRA210_CLK_CLK_MAX, 24576000, 0 }, { TEGRA210_CLK_I2S0_SYNC, TEGRA210_CLK_CLK_MAX, 24576000, 0 }, diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c index 4786e0ebc2e8..6cb863c13648 100644 --- a/drivers/clk/ti/divider.c +++ b/drivers/clk/ti/divider.c @@ -425,91 +425,6 @@ int ti_clk_parse_divider_data(int *div_table, int num_dividers, int max_div, return 0; } -static const struct clk_div_table * -_get_div_table_from_setup(struct ti_clk_divider *setup, u8 *width) -{ - const struct clk_div_table *table = NULL; - - ti_clk_parse_divider_data(setup->dividers, setup->num_dividers, - setup->max_div, setup->flags, width, - &table); - - return table; -} - -struct clk_hw *ti_clk_build_component_div(struct ti_clk_divider *setup) -{ - struct clk_omap_divider *div; - struct clk_omap_reg *reg; - int ret; - - if (!setup) - return NULL; - - div = kzalloc(sizeof(*div), GFP_KERNEL); - if (!div) - return ERR_PTR(-ENOMEM); - - reg = (struct clk_omap_reg *)&div->reg; - reg->index = setup->module; - reg->offset = setup->reg; - - if (setup->flags & CLKF_INDEX_STARTS_AT_ONE) - div->flags |= CLK_DIVIDER_ONE_BASED; - - if (setup->flags & CLKF_INDEX_POWER_OF_TWO) - div->flags |= CLK_DIVIDER_POWER_OF_TWO; - - div->table = _get_div_table_from_setup(setup, &div->width); - if (IS_ERR(div->table)) { - ret = PTR_ERR(div->table); - kfree(div); - return ERR_PTR(ret); - } - - - div->shift = setup->bit_shift; - div->latch = -EINVAL; - - return &div->hw; -} - -struct clk *ti_clk_register_divider(struct ti_clk *setup) -{ - struct ti_clk_divider *div = setup->data; - struct clk_omap_reg reg = { - .index = div->module, - .offset = div->reg, - }; - u8 width; - u32 flags = 0; - u8 div_flags = 0; - const struct clk_div_table *table; - struct clk *clk; - - if (div->flags & CLKF_INDEX_STARTS_AT_ONE) - div_flags |= CLK_DIVIDER_ONE_BASED; - - if (div->flags & CLKF_INDEX_POWER_OF_TWO) - div_flags |= CLK_DIVIDER_POWER_OF_TWO; - - if (div->flags & CLKF_SET_RATE_PARENT) - flags |= CLK_SET_RATE_PARENT; - - table = _get_div_table_from_setup(div, &width); - if (IS_ERR(table)) - return (struct clk *)table; - - clk = _register_divider(NULL, setup->name, div->parent, - flags, ®, div->bit_shift, - width, -EINVAL, div_flags, table); - - if (IS_ERR(clk)) - kfree(table); - - return clk; -} - static struct clk_div_table * __init ti_clk_get_div_table(struct device_node *node) { diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c index 504c0e91cdc7..42389558418c 100644 --- a/drivers/clk/ti/gate.c +++ b/drivers/clk/ti/gate.c @@ -131,36 +131,6 @@ static struct clk *_register_gate(struct device *dev, const char *name, return clk; } -struct clk_hw *ti_clk_build_component_gate(struct ti_clk_gate *setup) -{ - struct clk_hw_omap *gate; - struct clk_omap_reg *reg; - const struct clk_hw_omap_ops *ops = &clkhwops_wait; - - if (!setup) - return NULL; - - gate = kzalloc(sizeof(*gate), GFP_KERNEL); - if (!gate) - return ERR_PTR(-ENOMEM); - - reg = (struct clk_omap_reg *)&gate->enable_reg; - reg->index = setup->module; - reg->offset = setup->reg; - - gate->enable_bit = setup->bit_shift; - - if (setup->flags & CLKF_NO_WAIT) - ops = NULL; - - if (setup->flags & CLKF_INTERFACE) - ops = &clkhwops_iclk_wait; - - gate->ops = ops; - - return &gate->hw; -} - static void __init _of_ti_gate_clk_setup(struct device_node *node, const struct clk_ops *ops, const struct clk_hw_omap_ops *hw_ops) diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c index b7f9a4f068bf..0069e7cf3ebc 100644 --- a/drivers/clk/ti/mux.c +++ b/drivers/clk/ti/mux.c @@ -164,37 +164,6 @@ static struct clk *_register_mux(struct device *dev, const char *name, return clk; } -struct clk *ti_clk_register_mux(struct ti_clk *setup) -{ - struct ti_clk_mux *mux; - u32 flags; - u8 mux_flags = 0; - struct clk_omap_reg reg; - u32 mask; - - mux = setup->data; - flags = CLK_SET_RATE_NO_REPARENT; - - mask = mux->num_parents; - if (!(mux->flags & CLKF_INDEX_STARTS_AT_ONE)) - mask--; - - mask = (1 << fls(mask)) - 1; - reg.index = mux->module; - reg.offset = mux->reg; - reg.ptr = NULL; - - if (mux->flags & CLKF_INDEX_STARTS_AT_ONE) - mux_flags |= CLK_MUX_INDEX_ONE; - - if (mux->flags & CLKF_SET_RATE_PARENT) - flags |= CLK_SET_RATE_PARENT; - - return _register_mux(NULL, setup->name, mux->parents, mux->num_parents, - flags, ®, mux->bit_shift, mask, -EINVAL, - mux_flags, NULL); -} - /** * of_mux_clk_setup - Setup function for simple mux rate clock * @node: DT node for the clock diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c index 56a4ebbf00e0..f7c23fa468f0 100644 --- a/drivers/cpufreq/bmips-cpufreq.c +++ b/drivers/cpufreq/bmips-cpufreq.c @@ -131,23 +131,18 @@ static int bmips_cpufreq_exit(struct cpufreq_policy *policy) static int bmips_cpufreq_init(struct cpufreq_policy *policy) { struct cpufreq_frequency_table *freq_table; - int ret; freq_table = bmips_cpufreq_get_freq_table(policy); if (IS_ERR(freq_table)) { - ret = PTR_ERR(freq_table); - pr_err("%s: couldn't determine frequency table (%d).\n", - BMIPS_CPUFREQ_NAME, ret); - return ret; + pr_err("%s: couldn't determine frequency table (%ld).\n", + BMIPS_CPUFREQ_NAME, PTR_ERR(freq_table)); + return PTR_ERR(freq_table); } - ret = cpufreq_generic_init(policy, freq_table, TRANSITION_LATENCY); - if (ret) - bmips_cpufreq_exit(policy); - else - pr_info("%s: registered\n", BMIPS_CPUFREQ_NAME); + cpufreq_generic_init(policy, freq_table, TRANSITION_LATENCY); + pr_info("%s: registered\n", BMIPS_CPUFREQ_NAME); - return ret; + return 0; } static struct cpufreq_driver bmips_cpufreq_driver = { diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 0a9f675f2af4..8dda62367816 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -23,6 +23,7 @@ #include <linux/kernel_stat.h> #include <linux/module.h> #include <linux/mutex.h> +#include <linux/pm_qos.h> #include <linux/slab.h> #include <linux/suspend.h> #include <linux/syscore_ops.h> @@ -158,7 +159,7 @@ EXPORT_SYMBOL_GPL(arch_set_freq_scale); * - set policies transition latency * - policy->cpus with all possible CPUs */ -int cpufreq_generic_init(struct cpufreq_policy *policy, +void cpufreq_generic_init(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table, unsigned int transition_latency) { @@ -170,8 +171,6 @@ int cpufreq_generic_init(struct cpufreq_policy *policy, * share the clock and voltage and clock. */ cpumask_setall(policy->cpus); - - return 0; } EXPORT_SYMBOL_GPL(cpufreq_generic_init); @@ -714,23 +713,15 @@ static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) static ssize_t store_##file_name \ (struct cpufreq_policy *policy, const char *buf, size_t count) \ { \ - int ret, temp; \ - struct cpufreq_policy new_policy; \ + unsigned long val; \ + int ret; \ \ - memcpy(&new_policy, policy, sizeof(*policy)); \ - new_policy.min = policy->user_policy.min; \ - new_policy.max = policy->user_policy.max; \ - \ - ret = sscanf(buf, "%u", &new_policy.object); \ + ret = sscanf(buf, "%lu", &val); \ if (ret != 1) \ return -EINVAL; \ \ - temp = new_policy.object; \ - ret = cpufreq_set_policy(policy, &new_policy); \ - if (!ret) \ - policy->user_policy.object = temp; \ - \ - return ret ? ret : count; \ + ret = dev_pm_qos_update_request(policy->object##_freq_req, val);\ + return ret >= 0 ? count : ret; \ } store_one(scaling_min_freq, min); @@ -996,7 +987,7 @@ static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu) { struct device *dev = get_cpu_device(cpu); - if (!dev) + if (unlikely(!dev)) return; if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) @@ -1112,17 +1103,18 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cp return ret; } -static void refresh_frequency_limits(struct cpufreq_policy *policy) +void refresh_frequency_limits(struct cpufreq_policy *policy) { - struct cpufreq_policy new_policy = *policy; - - pr_debug("updating policy for CPU %u\n", policy->cpu); + struct cpufreq_policy new_policy; - new_policy.min = policy->user_policy.min; - new_policy.max = policy->user_policy.max; + if (!policy_is_inactive(policy)) { + new_policy = *policy; + pr_debug("updating policy for CPU %u\n", policy->cpu); - cpufreq_set_policy(policy, &new_policy); + cpufreq_set_policy(policy, &new_policy); + } } +EXPORT_SYMBOL(refresh_frequency_limits); static void handle_update(struct work_struct *work) { @@ -1130,14 +1122,60 @@ static void handle_update(struct work_struct *work) container_of(work, struct cpufreq_policy, update); pr_debug("handle_update for cpu %u called\n", policy->cpu); + down_write(&policy->rwsem); refresh_frequency_limits(policy); + up_write(&policy->rwsem); +} + +static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, + void *data) +{ + struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min); + + schedule_work(&policy->update); + return 0; +} + +static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq, + void *data) +{ + struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max); + + schedule_work(&policy->update); + return 0; +} + +static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) +{ + struct kobject *kobj; + struct completion *cmp; + + down_write(&policy->rwsem); + cpufreq_stats_free_table(policy); + kobj = &policy->kobj; + cmp = &policy->kobj_unregister; + up_write(&policy->rwsem); + kobject_put(kobj); + + /* + * We need to make sure that the underlying kobj is + * actually not referenced anymore by anybody before we + * proceed with unloading. + */ + pr_debug("waiting for dropping of refcount\n"); + wait_for_completion(cmp); + pr_debug("wait complete\n"); } static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) { struct cpufreq_policy *policy; + struct device *dev = get_cpu_device(cpu); int ret; + if (!dev) + return NULL; + policy = kzalloc(sizeof(*policy), GFP_KERNEL); if (!policy) return NULL; @@ -1154,7 +1192,7 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, cpufreq_global_kobject, "policy%u", cpu); if (ret) { - pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret); + dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret); /* * The entire policy object will be freed below, but the extra * memory allocated for the kobject name needs to be freed by @@ -1164,6 +1202,25 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) goto err_free_real_cpus; } + policy->nb_min.notifier_call = cpufreq_notifier_min; + policy->nb_max.notifier_call = cpufreq_notifier_max; + + ret = dev_pm_qos_add_notifier(dev, &policy->nb_min, + DEV_PM_QOS_MIN_FREQUENCY); + if (ret) { + dev_err(dev, "Failed to register MIN QoS notifier: %d (%*pbl)\n", + ret, cpumask_pr_args(policy->cpus)); + goto err_kobj_remove; + } + + ret = dev_pm_qos_add_notifier(dev, &policy->nb_max, + DEV_PM_QOS_MAX_FREQUENCY); + if (ret) { + dev_err(dev, "Failed to register MAX QoS notifier: %d (%*pbl)\n", + ret, cpumask_pr_args(policy->cpus)); + goto err_min_qos_notifier; + } + INIT_LIST_HEAD(&policy->policy_list); init_rwsem(&policy->rwsem); spin_lock_init(&policy->transition_lock); @@ -1174,6 +1231,11 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) policy->cpu = cpu; return policy; +err_min_qos_notifier: + dev_pm_qos_remove_notifier(dev, &policy->nb_min, + DEV_PM_QOS_MIN_FREQUENCY); +err_kobj_remove: + cpufreq_policy_put_kobj(policy); err_free_real_cpus: free_cpumask_var(policy->real_cpus); err_free_rcpumask: @@ -1186,30 +1248,9 @@ err_free_policy: return NULL; } -static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) -{ - struct kobject *kobj; - struct completion *cmp; - - down_write(&policy->rwsem); - cpufreq_stats_free_table(policy); - kobj = &policy->kobj; - cmp = &policy->kobj_unregister; - up_write(&policy->rwsem); - kobject_put(kobj); - - /* - * We need to make sure that the underlying kobj is - * actually not referenced anymore by anybody before we - * proceed with unloading. - */ - pr_debug("waiting for dropping of refcount\n"); - wait_for_completion(cmp); - pr_debug("wait complete\n"); -} - static void cpufreq_policy_free(struct cpufreq_policy *policy) { + struct device *dev = get_cpu_device(policy->cpu); unsigned long flags; int cpu; @@ -1221,6 +1262,14 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) per_cpu(cpufreq_cpu_data, cpu) = NULL; write_unlock_irqrestore(&cpufreq_driver_lock, flags); + dev_pm_qos_remove_notifier(dev, &policy->nb_max, + DEV_PM_QOS_MAX_FREQUENCY); + dev_pm_qos_remove_notifier(dev, &policy->nb_min, + DEV_PM_QOS_MIN_FREQUENCY); + dev_pm_qos_remove_request(policy->max_freq_req); + dev_pm_qos_remove_request(policy->min_freq_req); + kfree(policy->min_freq_req); + cpufreq_policy_put_kobj(policy); free_cpumask_var(policy->real_cpus); free_cpumask_var(policy->related_cpus); @@ -1298,16 +1347,50 @@ static int cpufreq_online(unsigned int cpu) cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); if (new_policy) { - policy->user_policy.min = policy->min; - policy->user_policy.max = policy->max; + struct device *dev = get_cpu_device(cpu); for_each_cpu(j, policy->related_cpus) { per_cpu(cpufreq_cpu_data, j) = policy; add_cpu_dev_symlink(policy, j); } - } else { - policy->min = policy->user_policy.min; - policy->max = policy->user_policy.max; + + policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req), + GFP_KERNEL); + if (!policy->min_freq_req) + goto out_destroy_policy; + + ret = dev_pm_qos_add_request(dev, policy->min_freq_req, + DEV_PM_QOS_MIN_FREQUENCY, + policy->min); + if (ret < 0) { + /* + * So we don't call dev_pm_qos_remove_request() for an + * uninitialized request. + */ + kfree(policy->min_freq_req); + policy->min_freq_req = NULL; + + dev_err(dev, "Failed to add min-freq constraint (%d)\n", + ret); + goto out_destroy_policy; + } + + /* + * This must be initialized right here to avoid calling + * dev_pm_qos_remove_request() on uninitialized request in case + * of errors. + */ + policy->max_freq_req = policy->min_freq_req + 1; + + ret = dev_pm_qos_add_request(dev, policy->max_freq_req, + DEV_PM_QOS_MAX_FREQUENCY, + policy->max); + if (ret < 0) { + policy->max_freq_req = NULL; + dev_err(dev, "Failed to add max-freq constraint (%d)\n", + ret); + goto out_destroy_policy; + } } if (cpufreq_driver->get && has_target()) { @@ -2280,6 +2363,7 @@ int cpufreq_set_policy(struct cpufreq_policy *policy, struct cpufreq_policy *new_policy) { struct cpufreq_governor *old_gov; + struct device *cpu_dev = get_cpu_device(policy->cpu); int ret; pr_debug("setting new policy for CPU %u: %u - %u kHz\n", @@ -2288,17 +2372,21 @@ int cpufreq_set_policy(struct cpufreq_policy *policy, memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); /* - * This check works well when we store new min/max freq attributes, - * because new_policy is a copy of policy with one field updated. - */ - if (new_policy->min > new_policy->max) - return -EINVAL; + * PM QoS framework collects all the requests from users and provide us + * the final aggregated value here. + */ + new_policy->min = dev_pm_qos_read_value(cpu_dev, DEV_PM_QOS_MIN_FREQUENCY); + new_policy->max = dev_pm_qos_read_value(cpu_dev, DEV_PM_QOS_MAX_FREQUENCY); /* verify the cpu speed can be set within this limit */ ret = cpufreq_driver->verify(new_policy); if (ret) return ret; + /* + * The notifier-chain shall be removed once all the users of + * CPUFREQ_ADJUST are moved to use the QoS framework. + */ /* adjust if necessary - all reasons */ blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_ADJUST, new_policy); @@ -2377,10 +2465,9 @@ int cpufreq_set_policy(struct cpufreq_policy *policy, * @cpu: CPU to re-evaluate the policy for. * * Update the current frequency for the cpufreq policy of @cpu and use - * cpufreq_set_policy() to re-apply the min and max limits saved in the - * user_policy sub-structure of that policy, which triggers the evaluation - * of policy notifiers and the cpufreq driver's ->verify() callback for the - * policy in question, among other things. + * cpufreq_set_policy() to re-apply the min and max limits, which triggers the + * evaluation of policy notifiers and the cpufreq driver's ->verify() callback + * for the policy in question, among other things. */ void cpufreq_update_policy(unsigned int cpu) { @@ -2440,10 +2527,9 @@ static int cpufreq_boost_set_sw(int state) break; } - down_write(&policy->rwsem); - policy->user_policy.max = policy->max; - cpufreq_governor_limits(policy); - up_write(&policy->rwsem); + ret = dev_pm_qos_update_request(policy->max_freq_req, policy->max); + if (ret) + break; } return ret; diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 3de48ae60c29..297d23cad8b5 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -90,7 +90,8 @@ static int davinci_cpu_init(struct cpufreq_policy *policy) * Setting the latency to 2000 us to accommodate addition of drivers * to pre/post change notification list. */ - return cpufreq_generic_init(policy, freq_table, 2000 * 1000); + cpufreq_generic_init(policy, freq_table, 2000 * 1000); + return 0; } static struct cpufreq_driver davinci_driver = { diff --git a/drivers/cpufreq/imx-cpufreq-dt.c b/drivers/cpufreq/imx-cpufreq-dt.c index b54fd26ea7df..4f85f3112784 100644 --- a/drivers/cpufreq/imx-cpufreq-dt.c +++ b/drivers/cpufreq/imx-cpufreq-dt.c @@ -44,10 +44,11 @@ static int imx_cpufreq_dt_probe(struct platform_device *pdev) * According to datasheet minimum speed grading is not supported for * consumer parts so clamp to 1 to avoid warning for "no OPPs" * - * Applies to 8mq and 8mm. + * Applies to i.MX8M series SoCs. */ if (mkt_segment == 0 && speed_grade == 0 && ( of_machine_is_compatible("fsl,imx8mm") || + of_machine_is_compatible("fsl,imx8mn") || of_machine_is_compatible("fsl,imx8mq"))) speed_grade = 1; diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index 47ccfa6b17b7..648a09a1778a 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -190,14 +190,12 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index) static int imx6q_cpufreq_init(struct cpufreq_policy *policy) { - int ret; - policy->clk = clks[ARM].clk; - ret = cpufreq_generic_init(policy, freq_table, transition_latency); + cpufreq_generic_init(policy, freq_table, transition_latency); policy->suspend_freq = max_freq; dev_pm_opp_of_register_em(policy->cpus); - return ret; + return 0; } static struct cpufreq_driver imx6q_cpufreq_driver = { diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index f2ff5de988c1..cc27d4c59dca 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -898,7 +898,6 @@ static void intel_pstate_update_policies(void) static void intel_pstate_update_max_freq(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); - struct cpufreq_policy new_policy; struct cpudata *cpudata; if (!policy) @@ -908,11 +907,7 @@ static void intel_pstate_update_max_freq(unsigned int cpu) policy->cpuinfo.max_freq = global.turbo_disabled_mf ? cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; - memcpy(&new_policy, policy, sizeof(*policy)); - new_policy.max = min(policy->user_policy.max, policy->cpuinfo.max_freq); - new_policy.min = min(policy->user_policy.min, new_policy.max); - - cpufreq_set_policy(policy, &new_policy); + refresh_frequency_limits(policy); cpufreq_cpu_release(policy); } diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c index 7ab564c1f7ae..cb74bdc5baaa 100644 --- a/drivers/cpufreq/kirkwood-cpufreq.c +++ b/drivers/cpufreq/kirkwood-cpufreq.c @@ -85,7 +85,8 @@ static int kirkwood_cpufreq_target(struct cpufreq_policy *policy, /* Module init and exit code */ static int kirkwood_cpufreq_cpu_init(struct cpufreq_policy *policy) { - return cpufreq_generic_init(policy, kirkwood_freq_table, 5000); + cpufreq_generic_init(policy, kirkwood_freq_table, 5000); + return 0; } static struct cpufreq_driver kirkwood_cpufreq_driver = { diff --git a/drivers/cpufreq/loongson1-cpufreq.c b/drivers/cpufreq/loongson1-cpufreq.c index 21c9ce8526c0..0ea88778882a 100644 --- a/drivers/cpufreq/loongson1-cpufreq.c +++ b/drivers/cpufreq/loongson1-cpufreq.c @@ -81,7 +81,7 @@ static int ls1x_cpufreq_init(struct cpufreq_policy *policy) struct device *cpu_dev = get_cpu_device(policy->cpu); struct cpufreq_frequency_table *freq_tbl; unsigned int pll_freq, freq; - int steps, i, ret; + int steps, i; pll_freq = clk_get_rate(cpufreq->pll_clk) / 1000; @@ -103,11 +103,9 @@ static int ls1x_cpufreq_init(struct cpufreq_policy *policy) freq_tbl[i].frequency = CPUFREQ_TABLE_END; policy->clk = cpufreq->clk; - ret = cpufreq_generic_init(policy, freq_tbl, 0); - if (ret) - kfree(freq_tbl); + cpufreq_generic_init(policy, freq_tbl, 0); - return ret; + return 0; } static int ls1x_cpufreq_exit(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c index da344696beed..890813e0bb76 100644 --- a/drivers/cpufreq/loongson2_cpufreq.c +++ b/drivers/cpufreq/loongson2_cpufreq.c @@ -95,7 +95,8 @@ static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy) } policy->clk = cpuclk; - return cpufreq_generic_init(policy, &loongson2_clockmod_table[0], 0); + cpufreq_generic_init(policy, &loongson2_clockmod_table[0], 0); + return 0; } static int loongson2_cpufreq_exit(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/maple-cpufreq.c b/drivers/cpufreq/maple-cpufreq.c index f5220b3d4ec5..28d346062166 100644 --- a/drivers/cpufreq/maple-cpufreq.c +++ b/drivers/cpufreq/maple-cpufreq.c @@ -140,7 +140,8 @@ static unsigned int maple_cpufreq_get_speed(unsigned int cpu) static int maple_cpufreq_cpu_init(struct cpufreq_policy *policy) { - return cpufreq_generic_init(policy, maple_cpu_freqs, 12000); + cpufreq_generic_init(policy, maple_cpu_freqs, 12000); + return 0; } static struct cpufreq_driver maple_cpufreq_driver = { diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index 29643f06a3c3..8d14b42a8c6f 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -122,23 +122,18 @@ static int omap_cpu_init(struct cpufreq_policy *policy) dev_err(mpu_dev, "%s: cpu%d: failed creating freq table[%d]\n", __func__, policy->cpu, result); - goto fail; + clk_put(policy->clk); + return result; } } atomic_inc_return(&freq_table_users); /* FIXME: what's the actual transition time? */ - result = cpufreq_generic_init(policy, freq_table, 300 * 1000); - if (!result) { - dev_pm_opp_of_register_em(policy->cpus); - return 0; - } + cpufreq_generic_init(policy, freq_table, 300 * 1000); + dev_pm_opp_of_register_em(policy->cpus); - freq_table_free(); -fail: - clk_put(policy->clk); - return result; + return 0; } static int omap_cpu_exit(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/pasemi-cpufreq.c b/drivers/cpufreq/pasemi-cpufreq.c index 6b1e4abe3248..93f39a1d4c3d 100644 --- a/drivers/cpufreq/pasemi-cpufreq.c +++ b/drivers/cpufreq/pasemi-cpufreq.c @@ -196,7 +196,8 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy) policy->cur = pas_freqs[cur_astate].frequency; ppc_proc_freq = policy->cur * 1000ul; - return cpufreq_generic_init(policy, pas_freqs, get_gizmo_latency()); + cpufreq_generic_init(policy, pas_freqs, get_gizmo_latency()); + return 0; out_unmap_sdcpwr: iounmap(sdcpwr_mapbase); diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c index 650104d729f3..73621bc11976 100644 --- a/drivers/cpufreq/pmac32-cpufreq.c +++ b/drivers/cpufreq/pmac32-cpufreq.c @@ -372,7 +372,8 @@ static int pmac_cpufreq_target( struct cpufreq_policy *policy, static int pmac_cpufreq_cpu_init(struct cpufreq_policy *policy) { - return cpufreq_generic_init(policy, pmac_cpu_freqs, transition_latency); + cpufreq_generic_init(policy, pmac_cpu_freqs, transition_latency); + return 0; } static u32 read_gpio(struct device_node *np) diff --git a/drivers/cpufreq/pmac64-cpufreq.c b/drivers/cpufreq/pmac64-cpufreq.c index 1af3492a000d..d7542a106e6b 100644 --- a/drivers/cpufreq/pmac64-cpufreq.c +++ b/drivers/cpufreq/pmac64-cpufreq.c @@ -321,7 +321,8 @@ static unsigned int g5_cpufreq_get_speed(unsigned int cpu) static int g5_cpufreq_cpu_init(struct cpufreq_policy *policy) { - return cpufreq_generic_init(policy, g5_cpu_freqs, transition_latency); + cpufreq_generic_init(policy, g5_cpu_freqs, transition_latency); + return 0; } static struct cpufreq_driver g5_cpufreq_driver = { diff --git a/drivers/cpufreq/s3c2416-cpufreq.c b/drivers/cpufreq/s3c2416-cpufreq.c index f7ff1ed7fef1..106910351c41 100644 --- a/drivers/cpufreq/s3c2416-cpufreq.c +++ b/drivers/cpufreq/s3c2416-cpufreq.c @@ -447,21 +447,16 @@ static int s3c2416_cpufreq_driver_init(struct cpufreq_policy *policy) /* Datasheet says PLL stabalisation time must be at least 300us, * so but add some fudge. (reference in LOCKCON0 register description) */ - ret = cpufreq_generic_init(policy, s3c_freq->freq_table, + cpufreq_generic_init(policy, s3c_freq->freq_table, (500 * 1000) + s3c_freq->regulator_latency); - if (ret) - goto err_freq_table; - register_reboot_notifier(&s3c2416_cpufreq_reboot_notifier); return 0; -err_freq_table: #ifdef CONFIG_ARM_S3C2416_CPUFREQ_VCORESCALE - regulator_put(s3c_freq->vddarm); err_vddarm: -#endif clk_put(s3c_freq->armclk); +#endif err_armclk: clk_put(s3c_freq->hclk); err_hclk: diff --git a/drivers/cpufreq/s3c64xx-cpufreq.c b/drivers/cpufreq/s3c64xx-cpufreq.c index 37df2d892eb0..af0c00dabb22 100644 --- a/drivers/cpufreq/s3c64xx-cpufreq.c +++ b/drivers/cpufreq/s3c64xx-cpufreq.c @@ -144,7 +144,6 @@ out: static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy) { - int ret; struct cpufreq_frequency_table *freq; if (policy->cpu != 0) @@ -165,8 +164,7 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy) #ifdef CONFIG_REGULATOR vddarm = regulator_get(NULL, "vddarm"); if (IS_ERR(vddarm)) { - ret = PTR_ERR(vddarm); - pr_err("Failed to obtain VDDARM: %d\n", ret); + pr_err("Failed to obtain VDDARM: %ld\n", PTR_ERR(vddarm)); pr_err("Only frequency scaling available\n"); vddarm = NULL; } else { @@ -196,16 +194,9 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy) * the PLLs, which we don't currently) is ~300us worst case, * but add some fudge. */ - ret = cpufreq_generic_init(policy, s3c64xx_freq_table, + cpufreq_generic_init(policy, s3c64xx_freq_table, (500 * 1000) + regulator_latency); - if (ret != 0) { - pr_err("Failed to configure frequency table: %d\n", - ret); - regulator_put(vddarm); - clk_put(policy->clk); - } - - return ret; + return 0; } static struct cpufreq_driver s3c64xx_cpufreq_driver = { diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index e5cb17d4be7b..5d10030f2560 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -541,7 +541,8 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) s5pv210_dram_conf[1].freq = clk_get_rate(dmc1_clk); policy->suspend_freq = SLEEP_FREQ; - return cpufreq_generic_init(policy, s5pv210_freq_table, 40000); + cpufreq_generic_init(policy, s5pv210_freq_table, 40000); + return 0; out_dmc1: clk_put(dmc0_clk); diff --git a/drivers/cpufreq/sa1100-cpufreq.c b/drivers/cpufreq/sa1100-cpufreq.c index ab5cab93e638..5c075ef6adc0 100644 --- a/drivers/cpufreq/sa1100-cpufreq.c +++ b/drivers/cpufreq/sa1100-cpufreq.c @@ -181,7 +181,8 @@ static int sa1100_target(struct cpufreq_policy *policy, unsigned int ppcr) static int __init sa1100_cpu_init(struct cpufreq_policy *policy) { - return cpufreq_generic_init(policy, sa11x0_freq_table, 0); + cpufreq_generic_init(policy, sa11x0_freq_table, 0); + return 0; } static struct cpufreq_driver sa1100_driver __refdata = { diff --git a/drivers/cpufreq/sa1110-cpufreq.c b/drivers/cpufreq/sa1110-cpufreq.c index dab54e051c0e..d9d04d935b3a 100644 --- a/drivers/cpufreq/sa1110-cpufreq.c +++ b/drivers/cpufreq/sa1110-cpufreq.c @@ -303,7 +303,8 @@ static int sa1110_target(struct cpufreq_policy *policy, unsigned int ppcr) static int __init sa1110_cpu_init(struct cpufreq_policy *policy) { - return cpufreq_generic_init(policy, sa11x0_freq_table, 0); + cpufreq_generic_init(policy, sa11x0_freq_table, 0); + return 0; } /* sa1110_driver needs __refdata because it must remain after init registers diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c index 4074e2615522..73bd8dc47074 100644 --- a/drivers/cpufreq/spear-cpufreq.c +++ b/drivers/cpufreq/spear-cpufreq.c @@ -153,8 +153,9 @@ static int spear_cpufreq_target(struct cpufreq_policy *policy, static int spear_cpufreq_init(struct cpufreq_policy *policy) { policy->clk = spear_cpufreq.clk; - return cpufreq_generic_init(policy, spear_cpufreq.freq_tbl, + cpufreq_generic_init(policy, spear_cpufreq.freq_tbl, spear_cpufreq.transition_latency); + return 0; } static struct cpufreq_driver spear_cpufreq_driver = { diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c index 3c32cc7b0671..f84ecd22f488 100644 --- a/drivers/cpufreq/tegra20-cpufreq.c +++ b/drivers/cpufreq/tegra20-cpufreq.c @@ -118,17 +118,11 @@ static int tegra_target(struct cpufreq_policy *policy, unsigned int index) static int tegra_cpu_init(struct cpufreq_policy *policy) { struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data(); - int ret; clk_prepare_enable(cpufreq->cpu_clk); /* FIXME: what's the actual transition time? */ - ret = cpufreq_generic_init(policy, freq_table, 300 * 1000); - if (ret) { - clk_disable_unprepare(cpufreq->cpu_clk); - return ret; - } - + cpufreq_generic_init(policy, freq_table, 300 * 1000); policy->clk = cpufreq->cpu_clk; policy->suspend_freq = freq_table[0].frequency; return 0; diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index 9fddf828a76f..2e3e14192bee 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -110,7 +110,7 @@ int cpuidle_governor_latency_req(unsigned int cpu) { int global_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY); struct device *device = get_cpu_device(cpu); - int device_req = dev_pm_qos_raw_read_value(device); + int device_req = dev_pm_qos_raw_resume_latency(device); return device_req < global_req ? device_req : global_req; } diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 2109cfe80219..8fafbeab510a 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -295,6 +295,22 @@ static ssize_t target_node_show(struct device *dev, } static DEVICE_ATTR_RO(target_node); +static unsigned long long dev_dax_resource(struct dev_dax *dev_dax) +{ + struct dax_region *dax_region = dev_dax->region; + + return dax_region->res.start; +} + +static ssize_t resource_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + + return sprintf(buf, "%#llx\n", dev_dax_resource(dev_dax)); +} +static DEVICE_ATTR_RO(resource); + static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -313,6 +329,8 @@ static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n) if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0) return 0; + if (a == &dev_attr_resource.attr) + return 0400; return a->mode; } @@ -320,6 +338,7 @@ static struct attribute *dev_dax_attributes[] = { &dev_attr_modalias.attr, &dev_attr_size.attr, &dev_attr_target_node.attr, + &dev_attr_resource.attr, NULL, }; @@ -388,7 +407,7 @@ struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id, * No 'host' or dax_operations since there is no access to this * device outside of mmap of the resulting character device. */ - dax_dev = alloc_dax(dev_dax, NULL, NULL); + dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC); if (!dax_dev) goto err; diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h index c915889d1769..6ccca3b890d6 100644 --- a/drivers/dax/dax-private.h +++ b/drivers/dax/dax-private.h @@ -43,6 +43,7 @@ struct dax_region { * @target_node: effective numa node if dev_dax memory range is onlined * @dev - device core * @pgmap - pgmap for memmap setup / lifetime (driver owned) + * @dax_mem_res: physical address range of hotadded DAX memory */ struct dev_dax { struct dax_region *region; @@ -50,6 +51,7 @@ struct dev_dax { int target_node; struct device dev; struct dev_pagemap pgmap; + struct resource *dax_kmem_res; }; static inline struct dev_dax *to_dev_dax(struct device *dev) diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index a02318c6d28a..3d0a7e702c94 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -66,23 +66,59 @@ int dev_dax_kmem_probe(struct device *dev) new_res->name = dev_name(dev); rc = add_memory(numa_node, new_res->start, resource_size(new_res)); - if (rc) + if (rc) { + release_resource(new_res); + kfree(new_res); return rc; + } + dev_dax->dax_kmem_res = new_res; return 0; } +#ifdef CONFIG_MEMORY_HOTREMOVE +static int dev_dax_kmem_remove(struct device *dev) +{ + struct dev_dax *dev_dax = to_dev_dax(dev); + struct resource *res = dev_dax->dax_kmem_res; + resource_size_t kmem_start = res->start; + resource_size_t kmem_size = resource_size(res); + int rc; + + /* + * We have one shot for removing memory, if some memory blocks were not + * offline prior to calling this function remove_memory() will fail, and + * there is no way to hotremove this memory until reboot because device + * unbind will succeed even if we return failure. + */ + rc = remove_memory(dev_dax->target_node, kmem_start, kmem_size); + if (rc) { + dev_err(dev, + "DAX region %pR cannot be hotremoved until the next reboot\n", + res); + return rc; + } + + /* Release and free dax resources */ + release_resource(res); + kfree(res); + dev_dax->dax_kmem_res = NULL; + + return 0; +} +#else static int dev_dax_kmem_remove(struct device *dev) { /* - * Purposely leak the request_mem_region() for the device-dax - * range and return '0' to ->remove() attempts. The removal of - * the device from the driver always succeeds, but the region - * is permanently pinned as reserved by the unreleased + * Without hotremove purposely leak the request_mem_region() for the + * device-dax range and return '0' to ->remove() attempts. The removal + * of the device from the driver always succeeds, but the region is + * permanently pinned as reserved by the unreleased * request_mem_region(). */ return 0; } +#endif /* CONFIG_MEMORY_HOTREMOVE */ static struct dax_device_driver device_dax_kmem_driver = { .drv = { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 4e5ae7e8b557..8ab12068eea3 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -195,6 +195,8 @@ enum dax_device_flags { DAXDEV_ALIVE, /* gate whether dax_flush() calls the low level flush routine */ DAXDEV_WRITE_CACHE, + /* flag to check if device supports synchronous flush */ + DAXDEV_SYNC, }; /** @@ -372,6 +374,18 @@ bool dax_write_cache_enabled(struct dax_device *dax_dev) } EXPORT_SYMBOL_GPL(dax_write_cache_enabled); +bool __dax_synchronous(struct dax_device *dax_dev) +{ + return test_bit(DAXDEV_SYNC, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(__dax_synchronous); + +void __set_dax_synchronous(struct dax_device *dax_dev) +{ + set_bit(DAXDEV_SYNC, &dax_dev->flags); +} +EXPORT_SYMBOL_GPL(__set_dax_synchronous); + bool dax_alive(struct dax_device *dax_dev) { lockdep_assert_held(&dax_srcu); @@ -526,7 +540,7 @@ static void dax_add_host(struct dax_device *dax_dev, const char *host) } struct dax_device *alloc_dax(void *private, const char *__host, - const struct dax_operations *ops) + const struct dax_operations *ops, unsigned long flags) { struct dax_device *dax_dev; const char *host; @@ -549,6 +563,9 @@ struct dax_device *alloc_dax(void *private, const char *__host, dax_add_host(dax_dev, host); dax_dev->ops = ops; dax_dev->private = private; + if (flags & DAXDEV_F_SYNC) + set_dax_synchronous(dax_dev); + return dax_dev; err_dev: diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 703275cc29de..03fa0c58cef3 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -103,6 +103,7 @@ config AXI_DMAC depends on MICROBLAZE || NIOS2 || ARCH_ZYNQ || ARCH_ZYNQMP || ARCH_SOCFPGA || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS + select REGMAP_MMIO help Enable support for the Analog Devices AXI-DMAC peripheral. This DMA controller is often used in Analog Device's reference designs for FPGA @@ -584,7 +585,7 @@ config TEGRA20_APB_DMA config TEGRA210_ADMA tristate "NVIDIA Tegra210 ADMA support" - depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) && PM_CLK + depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -666,6 +667,8 @@ source "drivers/dma/qcom/Kconfig" source "drivers/dma/dw/Kconfig" +source "drivers/dma/dw-edma/Kconfig" + source "drivers/dma/hsu/Kconfig" source "drivers/dma/sh/Kconfig" diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 6126e1c3a875..5bddf6f8790f 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_DMA_SUN4I) += sun4i-dma.o obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ obj-$(CONFIG_DW_DMAC_CORE) += dw/ +obj-$(CONFIG_DW_EDMA) += dw-edma/ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o obj-$(CONFIG_FSL_DMA) += fsldma.o obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index 464725dcad00..9adc7a2fa3d3 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -2508,9 +2508,8 @@ DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs); static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) { /* Expose a simple debugfs interface to view all clocks */ - (void) debugfs_create_file(dev_name(&pl08x->adev->dev), - S_IFREG | S_IRUGO, NULL, pl08x, - &pl08x_debugfs_fops); + debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, + NULL, pl08x, &pl08x_debugfs_fops); } #else diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 627ef3e5b312..b58ac720d9a1 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -1568,11 +1568,14 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) struct at_xdmac_desc *desc; struct dma_async_tx_descriptor *txd; - desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); - txd = &desc->tx_dma_desc; + if (!list_empty(&atchan->xfers_list)) { + desc = list_first_entry(&atchan->xfers_list, + struct at_xdmac_desc, xfer_node); + txd = &desc->tx_dma_desc; - if (txd->flags & DMA_PREP_INTERRUPT) - dmaengine_desc_get_callback_invoke(txd, NULL); + if (txd->flags & DMA_PREP_INTERRUPT) + dmaengine_desc_get_callback_invoke(txd, NULL); + } } static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index fa81d0177765..275e90fa829d 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -164,7 +164,6 @@ struct sba_device { struct list_head reqs_free_list; /* DebugFS directory entries */ struct dentry *root; - struct dentry *stats; }; /* ====== Command helper routines ===== */ @@ -1716,17 +1715,11 @@ static int sba_probe(struct platform_device *pdev) /* Create debugfs root entry */ sba->root = debugfs_create_dir(dev_name(sba->dev), NULL); - if (IS_ERR_OR_NULL(sba->root)) { - dev_err(sba->dev, "failed to create debugfs root entry\n"); - sba->root = NULL; - goto skip_debugfs; - } /* Create debugfs stats entry */ - sba->stats = debugfs_create_devm_seqfile(sba->dev, "stats", sba->root, - sba_debugfs_stats_show); - if (IS_ERR_OR_NULL(sba->stats)) - dev_err(sba->dev, "failed to create debugfs stats file\n"); + debugfs_create_devm_seqfile(sba->dev, "stats", sba->root, + sba_debugfs_stats_show); + skip_debugfs: /* Register DMA device with Linux async framework */ diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index 547786ac342b..e51d836afcc7 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -1378,10 +1378,8 @@ static int __init init_coh901318_debugfs(void) dma_dentry = debugfs_create_dir("dma", NULL); - (void) debugfs_create_file("status", - S_IFREG | S_IRUGO, - dma_dentry, NULL, - &coh901318_debugfs_status_operations); + debugfs_create_file("status", S_IFREG | S_IRUGO, dma_dentry, NULL, + &coh901318_debugfs_status_operations); return 0; } diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 8a3f1043917b..a0ee404b736e 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -2,7 +2,7 @@ /* * Driver for the Analog Devices AXI-DMAC core * - * Copyright 2013-2015 Analog Devices Inc. + * Copyright 2013-2019 Analog Devices Inc. * Author: Lars-Peter Clausen <lars@metafoo.de> */ @@ -18,7 +18,9 @@ #include <linux/of.h> #include <linux/of_dma.h> #include <linux/platform_device.h> +#include <linux/regmap.h> #include <linux/slab.h> +#include <linux/fpga/adi-axi-common.h> #include <dt-bindings/dma/axi-dmac.h> @@ -62,6 +64,8 @@ #define AXI_DMAC_REG_STATUS 0x430 #define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434 #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 +#define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c +#define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 #define AXI_DMAC_CTRL_ENABLE BIT(0) #define AXI_DMAC_CTRL_PAUSE BIT(1) @@ -70,6 +74,10 @@ #define AXI_DMAC_IRQ_EOT BIT(1) #define AXI_DMAC_FLAG_CYCLIC BIT(0) +#define AXI_DMAC_FLAG_LAST BIT(1) +#define AXI_DMAC_FLAG_PARTIAL_REPORT BIT(2) + +#define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31) /* The maximum ID allocated by the hardware is 31 */ #define AXI_DMAC_SG_UNUSED 32U @@ -82,12 +90,14 @@ struct axi_dmac_sg { unsigned int dest_stride; unsigned int src_stride; unsigned int id; + unsigned int partial_len; bool schedule_when_free; }; struct axi_dmac_desc { struct virt_dma_desc vdesc; bool cyclic; + bool have_partial_xfer; unsigned int num_submitted; unsigned int num_completed; @@ -108,8 +118,10 @@ struct axi_dmac_chan { unsigned int dest_type; unsigned int max_length; - unsigned int align_mask; + unsigned int address_align_mask; + unsigned int length_align_mask; + bool hw_partial_xfer; bool hw_cyclic; bool hw_2d; }; @@ -167,14 +179,14 @@ static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len) { if (len == 0) return false; - if ((len & chan->align_mask) != 0) /* Not aligned */ + if ((len & chan->length_align_mask) != 0) /* Not aligned */ return false; return true; } static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr) { - if ((addr & chan->align_mask) != 0) /* Not aligned */ + if ((addr & chan->address_align_mask) != 0) /* Not aligned */ return false; return true; } @@ -210,11 +222,13 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) } desc->num_submitted++; - if (desc->num_submitted == desc->num_sgs) { + if (desc->num_submitted == desc->num_sgs || + desc->have_partial_xfer) { if (desc->cyclic) desc->num_submitted = 0; /* Start again */ else chan->next_desc = NULL; + flags |= AXI_DMAC_FLAG_LAST; } else { chan->next_desc = desc; } @@ -240,6 +254,9 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) desc->num_sgs == 1) flags |= AXI_DMAC_FLAG_CYCLIC; + if (chan->hw_partial_xfer) + flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); @@ -252,6 +269,83 @@ static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) struct axi_dmac_desc, vdesc.node); } +static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, + struct axi_dmac_sg *sg) +{ + if (chan->hw_2d) + return sg->x_len * sg->y_len; + else + return sg->x_len; +} + +static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) +{ + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + struct axi_dmac_desc *desc; + struct axi_dmac_sg *sg; + u32 xfer_done, len, id, i; + bool found_sg; + + do { + len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN); + id = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID); + + found_sg = false; + list_for_each_entry(desc, &chan->active_descs, vdesc.node) { + for (i = 0; i < desc->num_sgs; i++) { + sg = &desc->sg[i]; + if (sg->id == AXI_DMAC_SG_UNUSED) + continue; + if (sg->id == id) { + desc->have_partial_xfer = true; + sg->partial_len = len; + found_sg = true; + break; + } + } + if (found_sg) + break; + } + + if (found_sg) { + dev_dbg(dmac->dma_dev.dev, + "Found partial segment id=%u, len=%u\n", + id, len); + } else { + dev_warn(dmac->dma_dev.dev, + "Not found partial segment id=%u, len=%u\n", + id, len); + } + + /* Check if we have any more partial transfers */ + xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); + xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE); + + } while (!xfer_done); +} + +static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, + struct axi_dmac_desc *active) +{ + struct dmaengine_result *rslt = &active->vdesc.tx_result; + unsigned int start = active->num_completed - 1; + struct axi_dmac_sg *sg; + unsigned int i, total; + + rslt->result = DMA_TRANS_NOERROR; + rslt->residue = 0; + + /* + * We get here if the last completed segment is partial, which + * means we can compute the residue from that segment onwards + */ + for (i = start; i < active->num_sgs; i++) { + sg = &active->sg[i]; + total = axi_dmac_total_sg_bytes(chan, sg); + rslt->residue += (total - sg->partial_len); + } +} + static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, unsigned int completed_transfers) { @@ -263,6 +357,10 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, if (!active) return false; + if (chan->hw_partial_xfer && + (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) + axi_dmac_dequeue_partial_xfers(chan); + do { sg = &active->sg[active->num_completed]; if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ @@ -276,10 +374,14 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, start_next = true; } + if (sg->partial_len) + axi_dmac_compute_residue(chan, active); + if (active->cyclic) vchan_cyclic_callback(&active->vdesc); - if (active->num_completed == active->num_sgs) { + if (active->num_completed == active->num_sgs || + sg->partial_len) { if (active->cyclic) { active->num_completed = 0; /* wrap around */ } else { @@ -391,7 +493,7 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, num_segments = DIV_ROUND_UP(period_len, chan->max_length); segment_size = DIV_ROUND_UP(period_len, num_segments); /* Take care of alignment */ - segment_size = ((segment_size - 1) | chan->align_mask) + 1; + segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; for (i = 0; i < num_periods; i++) { len = period_len; @@ -561,6 +663,9 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( desc->sg[0].y_len = 1; } + if (flags & DMA_CYCLIC) + desc->cyclic = true; + return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); } @@ -574,6 +679,44 @@ static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) kfree(container_of(vdesc, struct axi_dmac_desc, vdesc)); } +static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) +{ + switch (reg) { + case AXI_DMAC_REG_IRQ_MASK: + case AXI_DMAC_REG_IRQ_SOURCE: + case AXI_DMAC_REG_IRQ_PENDING: + case AXI_DMAC_REG_CTRL: + case AXI_DMAC_REG_TRANSFER_ID: + case AXI_DMAC_REG_START_TRANSFER: + case AXI_DMAC_REG_FLAGS: + case AXI_DMAC_REG_DEST_ADDRESS: + case AXI_DMAC_REG_SRC_ADDRESS: + case AXI_DMAC_REG_X_LENGTH: + case AXI_DMAC_REG_Y_LENGTH: + case AXI_DMAC_REG_DEST_STRIDE: + case AXI_DMAC_REG_SRC_STRIDE: + case AXI_DMAC_REG_TRANSFER_DONE: + case AXI_DMAC_REG_ACTIVE_TRANSFER_ID: + case AXI_DMAC_REG_STATUS: + case AXI_DMAC_REG_CURRENT_SRC_ADDR: + case AXI_DMAC_REG_CURRENT_DEST_ADDR: + case AXI_DMAC_REG_PARTIAL_XFER_LEN: + case AXI_DMAC_REG_PARTIAL_XFER_ID: + return true; + default: + return false; + } +} + +static const struct regmap_config axi_dmac_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID, + .readable_reg = axi_dmac_regmap_rdwr, + .writeable_reg = axi_dmac_regmap_rdwr, +}; + /* * The configuration stored in the devicetree matches the configuration * parameters of the peripheral instance and allows the driver to know which @@ -617,7 +760,7 @@ static int axi_dmac_parse_chan_dt(struct device_node *of_chan, return ret; chan->dest_width = val / 8; - chan->align_mask = max(chan->dest_width, chan->src_width) - 1; + chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1; if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) chan->direction = DMA_MEM_TO_MEM; @@ -631,9 +774,12 @@ static int axi_dmac_parse_chan_dt(struct device_node *of_chan, return 0; } -static void axi_dmac_detect_caps(struct axi_dmac *dmac) +static int axi_dmac_detect_caps(struct axi_dmac *dmac) { struct axi_dmac_chan *chan = &dmac->chan; + unsigned int version; + + version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC); if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) @@ -647,6 +793,35 @@ static void axi_dmac_detect_caps(struct axi_dmac *dmac) chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); if (chan->max_length != UINT_MAX) chan->max_length++; + + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff); + if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 && + chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) { + dev_err(dmac->dma_dev.dev, + "Destination memory-mapped interface not supported."); + return -ENODEV; + } + + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff); + if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 && + chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) { + dev_err(dmac->dma_dev.dev, + "Source memory-mapped interface not supported."); + return -ENODEV; + } + + if (version >= ADI_AXI_PCORE_VER(4, 2, 'a')) + chan->hw_partial_xfer = true; + + if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) { + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00); + chan->length_align_mask = + axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); + } else { + chan->length_align_mask = chan->address_align_mask; + } + + return 0; } static int axi_dmac_probe(struct platform_device *pdev) @@ -722,7 +897,11 @@ static int axi_dmac_probe(struct platform_device *pdev) if (ret < 0) return ret; - axi_dmac_detect_caps(dmac); + ret = axi_dmac_detect_caps(dmac); + if (ret) + goto err_clk_disable; + + dma_dev->copy_align = (dmac->chan.address_align_mask + 1); axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00); @@ -742,6 +921,8 @@ static int axi_dmac_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dmac); + devm_regmap_init_mmio(&pdev->dev, dmac->base, &axi_dmac_regmap_config); + return 0; err_unregister_of: diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 6b8c4c458e8a..7fe9309a876b 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -156,7 +156,6 @@ struct jz4780_dma_dev { }; struct jz4780_dma_filter_data { - struct device_node *of_node; uint32_t transfer_type; int channel; }; @@ -772,8 +771,6 @@ static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); struct jz4780_dma_filter_data *data = param; - if (jzdma->dma_device.dev->of_node != data->of_node) - return false; if (data->channel > -1) { if (data->channel != jzchan->id) @@ -797,7 +794,6 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, if (dma_spec->args_count != 2) return NULL; - data.of_node = ofdma->of_node; data.transfer_type = dma_spec->args[0]; data.channel = dma_spec->args[1]; @@ -822,7 +818,8 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, return dma_get_slave_channel( &jzdma->chan[data.channel].vchan.chan); } else { - return dma_request_channel(mask, jz4780_dma_filter_fn, &data); + return __dma_request_channel(&mask, jz4780_dma_filter_fn, &data, + ofdma->of_node); } } diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 58cbf9fd5a46..03ac4b96117c 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -61,7 +61,7 @@ static long dmaengine_ref_count; /* --- sysfs implementation --- */ /** - * dev_to_dma_chan - convert a device pointer to the its sysfs container object + * dev_to_dma_chan - convert a device pointer to its sysfs container object * @dev - device node * * Must be called under dma_list_mutex @@ -629,11 +629,13 @@ EXPORT_SYMBOL_GPL(dma_get_any_slave_channel); * @mask: capabilities that the channel must satisfy * @fn: optional callback to disposition available channels * @fn_param: opaque parameter to pass to dma_filter_fn + * @np: device node to look for DMA channels * * Returns pointer to appropriate DMA channel on success or NULL. */ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, - dma_filter_fn fn, void *fn_param) + dma_filter_fn fn, void *fn_param, + struct device_node *np) { struct dma_device *device, *_d; struct dma_chan *chan = NULL; @@ -641,6 +643,10 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, /* Find a channel */ mutex_lock(&dma_list_mutex); list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { + /* Finds a DMA controller with matching device node */ + if (np && device->dev->of_node && np != device->dev->of_node) + continue; + chan = find_candidate(device, mask, fn, fn_param); if (!IS_ERR(chan)) break; @@ -699,7 +705,7 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name) chan = acpi_dma_request_slave_chan_by_name(dev, name); if (chan) { - /* Valid channel found or requester need to be deferred */ + /* Valid channel found or requester needs to be deferred */ if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER) return chan; } @@ -757,7 +763,7 @@ struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask) if (!mask) return ERR_PTR(-ENODEV); - chan = __dma_request_channel(mask, NULL, NULL); + chan = __dma_request_channel(mask, NULL, NULL, NULL); if (!chan) { mutex_lock(&dma_list_mutex); if (list_empty(&dma_device_list)) diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index d0ad46e916a6..3d22ae8dca72 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -62,7 +62,7 @@ MODULE_PARM_DESC(pq_sources, static int timeout = 3000; module_param(timeout, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " - "Pass -1 for infinite timeout"); + "Pass 0xFFFFFFFF (4294967295) for maximum timeout"); static bool noverify; module_param(noverify, bool, S_IRUGO | S_IWUSR); @@ -94,7 +94,7 @@ MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default * @iterations: iterations before stopping test * @xor_sources: number of xor source buffers * @pq_sources: number of p+q source buffers - * @timeout: transfer timeout in msec, -1 for infinite timeout + * @timeout: transfer timeout in msec, 0 - 0xFFFFFFFF (4294967295) */ struct dmatest_params { unsigned int buf_size; @@ -105,7 +105,7 @@ struct dmatest_params { unsigned int iterations; unsigned int xor_sources; unsigned int pq_sources; - int timeout; + unsigned int timeout; bool noverify; bool norandom; int alignment; diff --git a/drivers/dma/dw-edma/Kconfig b/drivers/dma/dw-edma/Kconfig new file mode 100644 index 000000000000..7ff17b2db6a1 --- /dev/null +++ b/drivers/dma/dw-edma/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 + +config DW_EDMA + tristate "Synopsys DesignWare eDMA controller driver" + depends on PCI && PCI_MSI + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support the Synopsys DesignWare eDMA controller, normally + implemented on endpoints SoCs. + +config DW_EDMA_PCIE + tristate "Synopsys DesignWare eDMA PCIe driver" + depends on PCI && PCI_MSI + select DW_EDMA + help + Provides a glue-logic between the Synopsys DesignWare + eDMA controller and an endpoint PCIe device. This also serves + as a reference design to whom desires to use this IP. diff --git a/drivers/dma/dw-edma/Makefile b/drivers/dma/dw-edma/Makefile new file mode 100644 index 000000000000..8d45c0d5689d --- /dev/null +++ b/drivers/dma/dw-edma/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DW_EDMA) += dw-edma.o +dw-edma-$(CONFIG_DEBUG_FS) := dw-edma-v0-debugfs.o +dw-edma-objs := dw-edma-core.o \ + dw-edma-v0-core.o $(dw-edma-y) +obj-$(CONFIG_DW_EDMA_PCIE) += dw-edma-pcie.o diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c new file mode 100644 index 000000000000..ff392c01bad1 --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA core driver + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/pm_runtime.h> +#include <linux/dmaengine.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/dma/edma.h> +#include <linux/pci.h> + +#include "dw-edma-core.h" +#include "dw-edma-v0-core.h" +#include "../dmaengine.h" +#include "../virt-dma.h" + +static inline +struct device *dchan2dev(struct dma_chan *dchan) +{ + return &dchan->dev->device; +} + +static inline +struct device *chan2dev(struct dw_edma_chan *chan) +{ + return &chan->vc.chan.dev->device; +} + +static inline +struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct dw_edma_desc, vd); +} + +static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) +{ + struct dw_edma_burst *burst; + + burst = kzalloc(sizeof(*burst), GFP_NOWAIT); + if (unlikely(!burst)) + return NULL; + + INIT_LIST_HEAD(&burst->list); + if (chunk->burst) { + /* Create and add new element into the linked list */ + chunk->bursts_alloc++; + list_add_tail(&burst->list, &chunk->burst->list); + } else { + /* List head */ + chunk->bursts_alloc = 0; + chunk->burst = burst; + } + + return burst; +} + +static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) +{ + struct dw_edma_chan *chan = desc->chan; + struct dw_edma *dw = chan->chip->dw; + struct dw_edma_chunk *chunk; + + chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); + if (unlikely(!chunk)) + return NULL; + + INIT_LIST_HEAD(&chunk->list); + chunk->chan = chan; + /* Toggling change bit (CB) in each chunk, this is a mechanism to + * inform the eDMA HW block that this is a new linked list ready + * to be consumed. + * - Odd chunks originate CB equal to 0 + * - Even chunks originate CB equal to 1 + */ + chunk->cb = !(desc->chunks_alloc % 2); + chunk->ll_region.paddr = dw->ll_region.paddr + chan->ll_off; + chunk->ll_region.vaddr = dw->ll_region.vaddr + chan->ll_off; + + if (desc->chunk) { + /* Create and add new element into the linked list */ + desc->chunks_alloc++; + list_add_tail(&chunk->list, &desc->chunk->list); + if (!dw_edma_alloc_burst(chunk)) { + kfree(chunk); + return NULL; + } + } else { + /* List head */ + chunk->burst = NULL; + desc->chunks_alloc = 0; + desc->chunk = chunk; + } + + return chunk; +} + +static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) +{ + struct dw_edma_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (unlikely(!desc)) + return NULL; + + desc->chan = chan; + if (!dw_edma_alloc_chunk(desc)) { + kfree(desc); + return NULL; + } + + return desc; +} + +static void dw_edma_free_burst(struct dw_edma_chunk *chunk) +{ + struct dw_edma_burst *child, *_next; + + /* Remove all the list elements */ + list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { + list_del(&child->list); + kfree(child); + chunk->bursts_alloc--; + } + + /* Remove the list head */ + kfree(child); + chunk->burst = NULL; +} + +static void dw_edma_free_chunk(struct dw_edma_desc *desc) +{ + struct dw_edma_chunk *child, *_next; + + if (!desc->chunk) + return; + + /* Remove all the list elements */ + list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { + dw_edma_free_burst(child); + list_del(&child->list); + kfree(child); + desc->chunks_alloc--; + } + + /* Remove the list head */ + kfree(child); + desc->chunk = NULL; +} + +static void dw_edma_free_desc(struct dw_edma_desc *desc) +{ + dw_edma_free_chunk(desc); + kfree(desc); +} + +static void vchan_free_desc(struct virt_dma_desc *vdesc) +{ + dw_edma_free_desc(vd2dw_edma_desc(vdesc)); +} + +static void dw_edma_start_transfer(struct dw_edma_chan *chan) +{ + struct dw_edma_chunk *child; + struct dw_edma_desc *desc; + struct virt_dma_desc *vd; + + vd = vchan_next_desc(&chan->vc); + if (!vd) + return; + + desc = vd2dw_edma_desc(vd); + if (!desc) + return; + + child = list_first_entry_or_null(&desc->chunk->list, + struct dw_edma_chunk, list); + if (!child) + return; + + dw_edma_v0_core_start(child, !desc->xfer_sz); + desc->xfer_sz += child->ll_region.sz; + dw_edma_free_burst(child); + list_del(&child->list); + kfree(child); + desc->chunks_alloc--; +} + +static int dw_edma_device_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + memcpy(&chan->config, config, sizeof(*config)); + chan->configured = true; + + return 0; +} + +static int dw_edma_device_pause(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + int err = 0; + + if (!chan->configured) + err = -EPERM; + else if (chan->status != EDMA_ST_BUSY) + err = -EPERM; + else if (chan->request != EDMA_REQ_NONE) + err = -EPERM; + else + chan->request = EDMA_REQ_PAUSE; + + return err; +} + +static int dw_edma_device_resume(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + int err = 0; + + if (!chan->configured) { + err = -EPERM; + } else if (chan->status != EDMA_ST_PAUSE) { + err = -EPERM; + } else if (chan->request != EDMA_REQ_NONE) { + err = -EPERM; + } else { + chan->status = EDMA_ST_BUSY; + dw_edma_start_transfer(chan); + } + + return err; +} + +static int dw_edma_device_terminate_all(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + int err = 0; + LIST_HEAD(head); + + if (!chan->configured) { + /* Do nothing */ + } else if (chan->status == EDMA_ST_PAUSE) { + chan->status = EDMA_ST_IDLE; + chan->configured = false; + } else if (chan->status == EDMA_ST_IDLE) { + chan->configured = false; + } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { + /* + * The channel is in a false BUSY state, probably didn't + * receive or lost an interrupt + */ + chan->status = EDMA_ST_IDLE; + chan->configured = false; + } else if (chan->request > EDMA_REQ_PAUSE) { + err = -EPERM; + } else { + chan->request = EDMA_REQ_STOP; + } + + return err; +} + +static void dw_edma_device_issue_pending(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + if (chan->configured && chan->request == EDMA_REQ_NONE && + chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) { + chan->status = EDMA_ST_BUSY; + dw_edma_start_transfer(chan); + } + spin_unlock_irqrestore(&chan->vc.lock, flags); +} + +static enum dma_status +dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + struct dw_edma_desc *desc; + struct virt_dma_desc *vd; + unsigned long flags; + enum dma_status ret; + u32 residue = 0; + + ret = dma_cookie_status(dchan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) + ret = DMA_PAUSED; + + if (!txstate) + goto ret_residue; + + spin_lock_irqsave(&chan->vc.lock, flags); + vd = vchan_find_desc(&chan->vc, cookie); + if (vd) { + desc = vd2dw_edma_desc(vd); + if (desc) + residue = desc->alloc_sz - desc->xfer_sz; + } + spin_unlock_irqrestore(&chan->vc.lock, flags); + +ret_residue: + dma_set_residue(txstate, residue); + + return ret; +} + +static struct dma_async_tx_descriptor * +dw_edma_device_transfer(struct dw_edma_transfer *xfer) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); + enum dma_transfer_direction direction = xfer->direction; + phys_addr_t src_addr, dst_addr; + struct scatterlist *sg = NULL; + struct dw_edma_chunk *chunk; + struct dw_edma_burst *burst; + struct dw_edma_desc *desc; + u32 cnt; + int i; + + if ((direction == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) || + (direction == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ)) + return NULL; + + if (xfer->cyclic) { + if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) + return NULL; + } else { + if (xfer->xfer.sg.len < 1) + return NULL; + } + + if (!chan->configured) + return NULL; + + desc = dw_edma_alloc_desc(chan); + if (unlikely(!desc)) + goto err_alloc; + + chunk = dw_edma_alloc_chunk(desc); + if (unlikely(!chunk)) + goto err_alloc; + + src_addr = chan->config.src_addr; + dst_addr = chan->config.dst_addr; + + if (xfer->cyclic) { + cnt = xfer->xfer.cyclic.cnt; + } else { + cnt = xfer->xfer.sg.len; + sg = xfer->xfer.sg.sgl; + } + + for (i = 0; i < cnt; i++) { + if (!xfer->cyclic && !sg) + break; + + if (chunk->bursts_alloc == chan->ll_max) { + chunk = dw_edma_alloc_chunk(desc); + if (unlikely(!chunk)) + goto err_alloc; + } + + burst = dw_edma_alloc_burst(chunk); + if (unlikely(!burst)) + goto err_alloc; + + if (xfer->cyclic) + burst->sz = xfer->xfer.cyclic.len; + else + burst->sz = sg_dma_len(sg); + + chunk->ll_region.sz += burst->sz; + desc->alloc_sz += burst->sz; + + if (direction == DMA_DEV_TO_MEM) { + burst->sar = src_addr; + if (xfer->cyclic) { + burst->dar = xfer->xfer.cyclic.paddr; + } else { + burst->dar = sg_dma_address(sg); + /* Unlike the typical assumption by other + * drivers/IPs the peripheral memory isn't + * a FIFO memory, in this case, it's a + * linear memory and that why the source + * and destination addresses are increased + * by the same portion (data length) + */ + src_addr += sg_dma_len(sg); + } + } else { + burst->dar = dst_addr; + if (xfer->cyclic) { + burst->sar = xfer->xfer.cyclic.paddr; + } else { + burst->sar = sg_dma_address(sg); + /* Unlike the typical assumption by other + * drivers/IPs the peripheral memory isn't + * a FIFO memory, in this case, it's a + * linear memory and that why the source + * and destination addresses are increased + * by the same portion (data length) + */ + dst_addr += sg_dma_len(sg); + } + } + + if (!xfer->cyclic) + sg = sg_next(sg); + } + + return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); + +err_alloc: + if (desc) + dw_edma_free_desc(desc); + + return NULL; +} + +static struct dma_async_tx_descriptor * +dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct dw_edma_transfer xfer; + + xfer.dchan = dchan; + xfer.direction = direction; + xfer.xfer.sg.sgl = sgl; + xfer.xfer.sg.len = len; + xfer.flags = flags; + xfer.cyclic = false; + + return dw_edma_device_transfer(&xfer); +} + +static struct dma_async_tx_descriptor * +dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, + size_t len, size_t count, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct dw_edma_transfer xfer; + + xfer.dchan = dchan; + xfer.direction = direction; + xfer.xfer.cyclic.paddr = paddr; + xfer.xfer.cyclic.len = len; + xfer.xfer.cyclic.cnt = count; + xfer.flags = flags; + xfer.cyclic = true; + + return dw_edma_device_transfer(&xfer); +} + +static void dw_edma_done_interrupt(struct dw_edma_chan *chan) +{ + struct dw_edma_desc *desc; + struct virt_dma_desc *vd; + unsigned long flags; + + dw_edma_v0_core_clear_done_int(chan); + + spin_lock_irqsave(&chan->vc.lock, flags); + vd = vchan_next_desc(&chan->vc); + if (vd) { + switch (chan->request) { + case EDMA_REQ_NONE: + desc = vd2dw_edma_desc(vd); + if (desc->chunks_alloc) { + chan->status = EDMA_ST_BUSY; + dw_edma_start_transfer(chan); + } else { + list_del(&vd->node); + vchan_cookie_complete(vd); + chan->status = EDMA_ST_IDLE; + } + break; + + case EDMA_REQ_STOP: + list_del(&vd->node); + vchan_cookie_complete(vd); + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_IDLE; + break; + + case EDMA_REQ_PAUSE: + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_PAUSE; + break; + + default: + break; + } + } + spin_unlock_irqrestore(&chan->vc.lock, flags); +} + +static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) +{ + struct virt_dma_desc *vd; + unsigned long flags; + + dw_edma_v0_core_clear_abort_int(chan); + + spin_lock_irqsave(&chan->vc.lock, flags); + vd = vchan_next_desc(&chan->vc); + if (vd) { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + spin_unlock_irqrestore(&chan->vc.lock, flags); + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_IDLE; +} + +static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) +{ + struct dw_edma_irq *dw_irq = data; + struct dw_edma *dw = dw_irq->dw; + unsigned long total, pos, val; + unsigned long off; + u32 mask; + + if (write) { + total = dw->wr_ch_cnt; + off = 0; + mask = dw_irq->wr_mask; + } else { + total = dw->rd_ch_cnt; + off = dw->wr_ch_cnt; + mask = dw_irq->rd_mask; + } + + val = dw_edma_v0_core_status_done_int(dw, write ? + EDMA_DIR_WRITE : + EDMA_DIR_READ); + val &= mask; + for_each_set_bit(pos, &val, total) { + struct dw_edma_chan *chan = &dw->chan[pos + off]; + + dw_edma_done_interrupt(chan); + } + + val = dw_edma_v0_core_status_abort_int(dw, write ? + EDMA_DIR_WRITE : + EDMA_DIR_READ); + val &= mask; + for_each_set_bit(pos, &val, total) { + struct dw_edma_chan *chan = &dw->chan[pos + off]; + + dw_edma_abort_interrupt(chan); + } + + return IRQ_HANDLED; +} + +static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) +{ + return dw_edma_interrupt(irq, data, true); +} + +static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) +{ + return dw_edma_interrupt(irq, data, false); +} + +static irqreturn_t dw_edma_interrupt_common(int irq, void *data) +{ + dw_edma_interrupt(irq, data, true); + dw_edma_interrupt(irq, data, false); + + return IRQ_HANDLED; +} + +static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + if (chan->status != EDMA_ST_IDLE) + return -EBUSY; + + pm_runtime_get(chan->chip->dev); + + return 0; +} + +static void dw_edma_free_chan_resources(struct dma_chan *dchan) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(5000); + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + int ret; + + while (time_before(jiffies, timeout)) { + ret = dw_edma_device_terminate_all(dchan); + if (!ret) + break; + + if (time_after_eq(jiffies, timeout)) + return; + + cpu_relax(); + } + + pm_runtime_put(chan->chip->dev); +} + +static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, + u32 wr_alloc, u32 rd_alloc) +{ + struct dw_edma_region *dt_region; + struct device *dev = chip->dev; + struct dw_edma *dw = chip->dw; + struct dw_edma_chan *chan; + size_t ll_chunk, dt_chunk; + struct dw_edma_irq *irq; + struct dma_device *dma; + u32 i, j, cnt, ch_cnt; + u32 alloc, off_alloc; + int err = 0; + u32 pos; + + ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; + ll_chunk = dw->ll_region.sz; + dt_chunk = dw->dt_region.sz; + + /* Calculate linked list chunk for each channel */ + ll_chunk /= roundup_pow_of_two(ch_cnt); + + /* Calculate linked list chunk for each channel */ + dt_chunk /= roundup_pow_of_two(ch_cnt); + + if (write) { + i = 0; + cnt = dw->wr_ch_cnt; + dma = &dw->wr_edma; + alloc = wr_alloc; + off_alloc = 0; + } else { + i = dw->wr_ch_cnt; + cnt = dw->rd_ch_cnt; + dma = &dw->rd_edma; + alloc = rd_alloc; + off_alloc = wr_alloc; + } + + INIT_LIST_HEAD(&dma->channels); + for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) { + chan = &dw->chan[i]; + + dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL); + if (!dt_region) + return -ENOMEM; + + chan->vc.chan.private = dt_region; + + chan->chip = chip; + chan->id = j; + chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ; + chan->configured = false; + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_IDLE; + + chan->ll_off = (ll_chunk * i); + chan->ll_max = (ll_chunk / EDMA_LL_SZ) - 1; + + chan->dt_off = (dt_chunk * i); + + dev_vdbg(dev, "L. List:\tChannel %s[%u] off=0x%.8lx, max_cnt=%u\n", + write ? "write" : "read", j, + chan->ll_off, chan->ll_max); + + if (dw->nr_irqs == 1) + pos = 0; + else + pos = off_alloc + (j % alloc); + + irq = &dw->irq[pos]; + + if (write) + irq->wr_mask |= BIT(j); + else + irq->rd_mask |= BIT(j); + + irq->dw = dw; + memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); + + dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", + write ? "write" : "read", j, + chan->msi.address_hi, chan->msi.address_lo, + chan->msi.data); + + chan->vc.desc_free = vchan_free_desc; + vchan_init(&chan->vc, dma); + + dt_region->paddr = dw->dt_region.paddr + chan->dt_off; + dt_region->vaddr = dw->dt_region.vaddr + chan->dt_off; + dt_region->sz = dt_chunk; + + dev_vdbg(dev, "Data:\tChannel %s[%u] off=0x%.8lx\n", + write ? "write" : "read", j, chan->dt_off); + + dw_edma_v0_core_device_config(chan); + } + + /* Set DMA channel capabilities */ + dma_cap_zero(dma->cap_mask); + dma_cap_set(DMA_SLAVE, dma->cap_mask); + dma_cap_set(DMA_CYCLIC, dma->cap_mask); + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV); + dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma->chancnt = cnt; + + /* Set DMA channel callbacks */ + dma->dev = chip->dev; + dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; + dma->device_free_chan_resources = dw_edma_free_chan_resources; + dma->device_config = dw_edma_device_config; + dma->device_pause = dw_edma_device_pause; + dma->device_resume = dw_edma_device_resume; + dma->device_terminate_all = dw_edma_device_terminate_all; + dma->device_issue_pending = dw_edma_device_issue_pending; + dma->device_tx_status = dw_edma_device_tx_status; + dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; + dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; + + dma_set_max_seg_size(dma->dev, U32_MAX); + + /* Register DMA device */ + err = dma_async_device_register(dma); + + return err; +} + +static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) +{ + if (*nr_irqs && *alloc < cnt) { + (*alloc)++; + (*nr_irqs)--; + } +} + +static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) +{ + while (*mask * alloc < cnt) + (*mask)++; +} + +static int dw_edma_irq_request(struct dw_edma_chip *chip, + u32 *wr_alloc, u32 *rd_alloc) +{ + struct device *dev = chip->dev; + struct dw_edma *dw = chip->dw; + u32 wr_mask = 1; + u32 rd_mask = 1; + int i, err = 0; + u32 ch_cnt; + + ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; + + if (dw->nr_irqs < 1) + return -EINVAL; + + if (dw->nr_irqs == 1) { + /* Common IRQ shared among all channels */ + err = request_irq(pci_irq_vector(to_pci_dev(dev), 0), + dw_edma_interrupt_common, + IRQF_SHARED, dw->name, &dw->irq[0]); + if (err) { + dw->nr_irqs = 0; + return err; + } + + get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), 0), + &dw->irq[0].msi); + } else { + /* Distribute IRQs equally among all channels */ + int tmp = dw->nr_irqs; + + while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { + dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); + dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); + } + + dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); + dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); + + for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { + err = request_irq(pci_irq_vector(to_pci_dev(dev), i), + i < *wr_alloc ? + dw_edma_interrupt_write : + dw_edma_interrupt_read, + IRQF_SHARED, dw->name, + &dw->irq[i]); + if (err) { + dw->nr_irqs = i; + return err; + } + + get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), i), + &dw->irq[i].msi); + } + + dw->nr_irqs = i; + } + + return err; +} + +int dw_edma_probe(struct dw_edma_chip *chip) +{ + struct device *dev = chip->dev; + struct dw_edma *dw = chip->dw; + u32 wr_alloc = 0; + u32 rd_alloc = 0; + int i, err; + + raw_spin_lock_init(&dw->lock); + + /* Find out how many write channels are supported by hardware */ + dw->wr_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE); + if (!dw->wr_ch_cnt) + return -EINVAL; + + /* Find out how many read channels are supported by hardware */ + dw->rd_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ); + if (!dw->rd_ch_cnt) + return -EINVAL; + + dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n", + dw->wr_ch_cnt, dw->rd_ch_cnt); + + /* Allocate channels */ + dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt, + sizeof(*dw->chan), GFP_KERNEL); + if (!dw->chan) + return -ENOMEM; + + snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id); + + /* Disable eDMA, only to establish the ideal initial conditions */ + dw_edma_v0_core_off(dw); + + /* Request IRQs */ + err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc); + if (err) + return err; + + /* Setup write channels */ + err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc); + if (err) + goto err_irq_free; + + /* Setup read channels */ + err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc); + if (err) + goto err_irq_free; + + /* Power management */ + pm_runtime_enable(dev); + + /* Turn debugfs on */ + dw_edma_v0_core_debugfs_on(chip); + + return 0; + +err_irq_free: + for (i = (dw->nr_irqs - 1); i >= 0; i--) + free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]); + + dw->nr_irqs = 0; + + return err; +} +EXPORT_SYMBOL_GPL(dw_edma_probe); + +int dw_edma_remove(struct dw_edma_chip *chip) +{ + struct dw_edma_chan *chan, *_chan; + struct device *dev = chip->dev; + struct dw_edma *dw = chip->dw; + int i; + + /* Disable eDMA */ + dw_edma_v0_core_off(dw); + + /* Free irqs */ + for (i = (dw->nr_irqs - 1); i >= 0; i--) + free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]); + + /* Power management */ + pm_runtime_disable(dev); + + list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, + vc.chan.device_node) { + list_del(&chan->vc.chan.device_node); + tasklet_kill(&chan->vc.task); + } + + list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels, + vc.chan.device_node) { + list_del(&chan->vc.chan.device_node); + tasklet_kill(&chan->vc.task); + } + + /* Deregister eDMA device */ + dma_async_device_unregister(&dw->wr_edma); + dma_async_device_unregister(&dw->rd_edma); + + /* Turn debugfs off */ + dw_edma_v0_core_debugfs_off(); + + return 0; +} +EXPORT_SYMBOL_GPL(dw_edma_remove); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver"); +MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h new file mode 100644 index 000000000000..b6cc90cbc9dc --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA core driver + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#ifndef _DW_EDMA_CORE_H +#define _DW_EDMA_CORE_H + +#include <linux/msi.h> +#include <linux/dma/edma.h> + +#include "../virt-dma.h" + +#define EDMA_LL_SZ 24 + +enum dw_edma_dir { + EDMA_DIR_WRITE = 0, + EDMA_DIR_READ +}; + +enum dw_edma_mode { + EDMA_MODE_LEGACY = 0, + EDMA_MODE_UNROLL +}; + +enum dw_edma_request { + EDMA_REQ_NONE = 0, + EDMA_REQ_STOP, + EDMA_REQ_PAUSE +}; + +enum dw_edma_status { + EDMA_ST_IDLE = 0, + EDMA_ST_PAUSE, + EDMA_ST_BUSY +}; + +struct dw_edma_chan; +struct dw_edma_chunk; + +struct dw_edma_burst { + struct list_head list; + u64 sar; + u64 dar; + u32 sz; +}; + +struct dw_edma_region { + phys_addr_t paddr; + dma_addr_t vaddr; + size_t sz; +}; + +struct dw_edma_chunk { + struct list_head list; + struct dw_edma_chan *chan; + struct dw_edma_burst *burst; + + u32 bursts_alloc; + + u8 cb; + struct dw_edma_region ll_region; /* Linked list */ +}; + +struct dw_edma_desc { + struct virt_dma_desc vd; + struct dw_edma_chan *chan; + struct dw_edma_chunk *chunk; + + u32 chunks_alloc; + + u32 alloc_sz; + u32 xfer_sz; +}; + +struct dw_edma_chan { + struct virt_dma_chan vc; + struct dw_edma_chip *chip; + int id; + enum dw_edma_dir dir; + + off_t ll_off; + u32 ll_max; + + off_t dt_off; + + struct msi_msg msi; + + enum dw_edma_request request; + enum dw_edma_status status; + u8 configured; + + struct dma_slave_config config; +}; + +struct dw_edma_irq { + struct msi_msg msi; + u32 wr_mask; + u32 rd_mask; + struct dw_edma *dw; +}; + +struct dw_edma { + char name[20]; + + struct dma_device wr_edma; + u16 wr_ch_cnt; + + struct dma_device rd_edma; + u16 rd_ch_cnt; + + struct dw_edma_region rg_region; /* Registers */ + struct dw_edma_region ll_region; /* Linked list */ + struct dw_edma_region dt_region; /* Data */ + + struct dw_edma_irq *irq; + int nr_irqs; + + u32 version; + enum dw_edma_mode mode; + + struct dw_edma_chan *chan; + const struct dw_edma_core_ops *ops; + + raw_spinlock_t lock; /* Only for legacy */ +}; + +struct dw_edma_sg { + struct scatterlist *sgl; + unsigned int len; +}; + +struct dw_edma_cyclic { + dma_addr_t paddr; + size_t len; + size_t cnt; +}; + +struct dw_edma_transfer { + struct dma_chan *dchan; + union dw_edma_xfer { + struct dw_edma_sg sg; + struct dw_edma_cyclic cyclic; + } xfer; + enum dma_transfer_direction direction; + unsigned long flags; + bool cyclic; +}; + +static inline +struct dw_edma_chan *vc2dw_edma_chan(struct virt_dma_chan *vc) +{ + return container_of(vc, struct dw_edma_chan, vc); +} + +static inline +struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan) +{ + return vc2dw_edma_chan(to_virt_chan(dchan)); +} + +#endif /* _DW_EDMA_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c new file mode 100644 index 000000000000..4c96e1c948f2 --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA PCIe driver + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/device.h> +#include <linux/dma/edma.h> +#include <linux/pci-epf.h> +#include <linux/msi.h> + +#include "dw-edma-core.h" + +struct dw_edma_pcie_data { + /* eDMA registers location */ + enum pci_barno rg_bar; + off_t rg_off; + size_t rg_sz; + /* eDMA memory linked list location */ + enum pci_barno ll_bar; + off_t ll_off; + size_t ll_sz; + /* eDMA memory data location */ + enum pci_barno dt_bar; + off_t dt_off; + size_t dt_sz; + /* Other */ + u32 version; + enum dw_edma_mode mode; + u8 irqs; +}; + +static const struct dw_edma_pcie_data snps_edda_data = { + /* eDMA registers location */ + .rg_bar = BAR_0, + .rg_off = 0x00001000, /* 4 Kbytes */ + .rg_sz = 0x00002000, /* 8 Kbytes */ + /* eDMA memory linked list location */ + .ll_bar = BAR_2, + .ll_off = 0x00000000, /* 0 Kbytes */ + .ll_sz = 0x00800000, /* 8 Mbytes */ + /* eDMA memory data location */ + .dt_bar = BAR_2, + .dt_off = 0x00800000, /* 8 Mbytes */ + .dt_sz = 0x03800000, /* 56 Mbytes */ + /* Other */ + .version = 0, + .mode = EDMA_MODE_UNROLL, + .irqs = 1, +}; + +static int dw_edma_pcie_probe(struct pci_dev *pdev, + const struct pci_device_id *pid) +{ + const struct dw_edma_pcie_data *pdata = (void *)pid->driver_data; + struct device *dev = &pdev->dev; + struct dw_edma_chip *chip; + int err, nr_irqs; + struct dw_edma *dw; + + /* Enable PCI device */ + err = pcim_enable_device(pdev); + if (err) { + pci_err(pdev, "enabling device failed\n"); + return err; + } + + /* Mapping PCI BAR regions */ + err = pcim_iomap_regions(pdev, BIT(pdata->rg_bar) | + BIT(pdata->ll_bar) | + BIT(pdata->dt_bar), + pci_name(pdev)); + if (err) { + pci_err(pdev, "eDMA BAR I/O remapping failed\n"); + return err; + } + + pci_set_master(pdev); + + /* DMA configuration */ + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + if (!err) { + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + if (err) { + pci_err(pdev, "consistent DMA mask 64 set failed\n"); + return err; + } + } else { + pci_err(pdev, "DMA mask 64 set failed\n"); + + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); + if (err) { + pci_err(pdev, "DMA mask 32 set failed\n"); + return err; + } + + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + if (err) { + pci_err(pdev, "consistent DMA mask 32 set failed\n"); + return err; + } + } + + /* Data structure allocation */ + chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + /* IRQs allocation */ + nr_irqs = pci_alloc_irq_vectors(pdev, 1, pdata->irqs, + PCI_IRQ_MSI | PCI_IRQ_MSIX); + if (nr_irqs < 1) { + pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n", + nr_irqs); + return -EPERM; + } + + /* Data structure initialization */ + chip->dw = dw; + chip->dev = dev; + chip->id = pdev->devfn; + chip->irq = pdev->irq; + + dw->rg_region.vaddr = (dma_addr_t)pcim_iomap_table(pdev)[pdata->rg_bar]; + dw->rg_region.vaddr += pdata->rg_off; + dw->rg_region.paddr = pdev->resource[pdata->rg_bar].start; + dw->rg_region.paddr += pdata->rg_off; + dw->rg_region.sz = pdata->rg_sz; + + dw->ll_region.vaddr = (dma_addr_t)pcim_iomap_table(pdev)[pdata->ll_bar]; + dw->ll_region.vaddr += pdata->ll_off; + dw->ll_region.paddr = pdev->resource[pdata->ll_bar].start; + dw->ll_region.paddr += pdata->ll_off; + dw->ll_region.sz = pdata->ll_sz; + + dw->dt_region.vaddr = (dma_addr_t)pcim_iomap_table(pdev)[pdata->dt_bar]; + dw->dt_region.vaddr += pdata->dt_off; + dw->dt_region.paddr = pdev->resource[pdata->dt_bar].start; + dw->dt_region.paddr += pdata->dt_off; + dw->dt_region.sz = pdata->dt_sz; + + dw->version = pdata->version; + dw->mode = pdata->mode; + dw->nr_irqs = nr_irqs; + + /* Debug info */ + pci_dbg(pdev, "Version:\t%u\n", dw->version); + + pci_dbg(pdev, "Mode:\t%s\n", + dw->mode == EDMA_MODE_LEGACY ? "Legacy" : "Unroll"); + + pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%pa, p=%pa)\n", + pdata->rg_bar, pdata->rg_off, pdata->rg_sz, + &dw->rg_region.vaddr, &dw->rg_region.paddr); + + pci_dbg(pdev, "L. List:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%pa, p=%pa)\n", + pdata->ll_bar, pdata->ll_off, pdata->ll_sz, + &dw->ll_region.vaddr, &dw->ll_region.paddr); + + pci_dbg(pdev, "Data:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%pa, p=%pa)\n", + pdata->dt_bar, pdata->dt_off, pdata->dt_sz, + &dw->dt_region.vaddr, &dw->dt_region.paddr); + + pci_dbg(pdev, "Nr. IRQs:\t%u\n", dw->nr_irqs); + + /* Validating if PCI interrupts were enabled */ + if (!pci_dev_msi_enabled(pdev)) { + pci_err(pdev, "enable interrupt failed\n"); + return -EPERM; + } + + dw->irq = devm_kcalloc(dev, nr_irqs, sizeof(*dw->irq), GFP_KERNEL); + if (!dw->irq) + return -ENOMEM; + + /* Starting eDMA driver */ + err = dw_edma_probe(chip); + if (err) { + pci_err(pdev, "eDMA probe failed\n"); + return err; + } + + /* Saving data structure reference */ + pci_set_drvdata(pdev, chip); + + return 0; +} + +static void dw_edma_pcie_remove(struct pci_dev *pdev) +{ + struct dw_edma_chip *chip = pci_get_drvdata(pdev); + int err; + + /* Stopping eDMA driver */ + err = dw_edma_remove(chip); + if (err) + pci_warn(pdev, "can't remove device properly: %d\n", err); + + /* Freeing IRQs */ + pci_free_irq_vectors(pdev); +} + +static const struct pci_device_id dw_edma_pcie_id_table[] = { + { PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) }, + { } +}; +MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table); + +static struct pci_driver dw_edma_pcie_driver = { + .name = "dw-edma-pcie", + .id_table = dw_edma_pcie_id_table, + .probe = dw_edma_pcie_probe, + .remove = dw_edma_pcie_remove, +}; + +module_pci_driver(dw_edma_pcie_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Synopsys DesignWare eDMA PCIe driver"); +MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>"); diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c new file mode 100644 index 000000000000..8a3180ed49a6 --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA v0 core + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#include <linux/bitfield.h> + +#include "dw-edma-core.h" +#include "dw-edma-v0-core.h" +#include "dw-edma-v0-regs.h" +#include "dw-edma-v0-debugfs.h" + +enum dw_edma_control { + DW_EDMA_V0_CB = BIT(0), + DW_EDMA_V0_TCB = BIT(1), + DW_EDMA_V0_LLP = BIT(2), + DW_EDMA_V0_LIE = BIT(3), + DW_EDMA_V0_RIE = BIT(4), + DW_EDMA_V0_CCS = BIT(8), + DW_EDMA_V0_LLE = BIT(9), +}; + +static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) +{ + return (struct dw_edma_v0_regs __iomem *)dw->rg_region.vaddr; +} + +#define SET(dw, name, value) \ + writel(value, &(__dw_regs(dw)->name)) + +#define GET(dw, name) \ + readl(&(__dw_regs(dw)->name)) + +#define SET_RW(dw, dir, name, value) \ + do { \ + if ((dir) == EDMA_DIR_WRITE) \ + SET(dw, wr_##name, value); \ + else \ + SET(dw, rd_##name, value); \ + } while (0) + +#define GET_RW(dw, dir, name) \ + ((dir) == EDMA_DIR_WRITE \ + ? GET(dw, wr_##name) \ + : GET(dw, rd_##name)) + +#define SET_BOTH(dw, name, value) \ + do { \ + SET(dw, wr_##name, value); \ + SET(dw, rd_##name, value); \ + } while (0) + +static inline struct dw_edma_v0_ch_regs __iomem * +__dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) +{ + if (dw->mode == EDMA_MODE_LEGACY) + return &(__dw_regs(dw)->type.legacy.ch); + + if (dir == EDMA_DIR_WRITE) + return &__dw_regs(dw)->type.unroll.ch[ch].wr; + + return &__dw_regs(dw)->type.unroll.ch[ch].rd; +} + +static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, + u32 value, void __iomem *addr) +{ + if (dw->mode == EDMA_MODE_LEGACY) { + u32 viewport_sel; + unsigned long flags; + + raw_spin_lock_irqsave(&dw->lock, flags); + + viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); + if (dir == EDMA_DIR_READ) + viewport_sel |= BIT(31); + + writel(viewport_sel, + &(__dw_regs(dw)->type.legacy.viewport_sel)); + writel(value, addr); + + raw_spin_unlock_irqrestore(&dw->lock, flags); + } else { + writel(value, addr); + } +} + +static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, + const void __iomem *addr) +{ + u32 value; + + if (dw->mode == EDMA_MODE_LEGACY) { + u32 viewport_sel; + unsigned long flags; + + raw_spin_lock_irqsave(&dw->lock, flags); + + viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); + if (dir == EDMA_DIR_READ) + viewport_sel |= BIT(31); + + writel(viewport_sel, + &(__dw_regs(dw)->type.legacy.viewport_sel)); + value = readl(addr); + + raw_spin_unlock_irqrestore(&dw->lock, flags); + } else { + value = readl(addr); + } + + return value; +} + +#define SET_CH(dw, dir, ch, name, value) \ + writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name)) + +#define GET_CH(dw, dir, ch, name) \ + readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) + +#define SET_LL(ll, value) \ + writel(value, ll) + +/* eDMA management callbacks */ +void dw_edma_v0_core_off(struct dw_edma *dw) +{ + SET_BOTH(dw, int_mask, EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + SET_BOTH(dw, int_clear, EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + SET_BOTH(dw, engine_en, 0); +} + +u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) +{ + u32 num_ch; + + if (dir == EDMA_DIR_WRITE) + num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK, GET(dw, ctrl)); + else + num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK, GET(dw, ctrl)); + + if (num_ch > EDMA_V0_MAX_NR_CH) + num_ch = EDMA_V0_MAX_NR_CH; + + return (u16)num_ch; +} + +enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->chip->dw; + u32 tmp; + + tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK, + GET_CH(dw, chan->dir, chan->id, ch_control1)); + + if (tmp == 1) + return DMA_IN_PROGRESS; + else if (tmp == 3) + return DMA_COMPLETE; + else + return DMA_ERROR; +} + +void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->chip->dw; + + SET_RW(dw, chan->dir, int_clear, + FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id))); +} + +void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->chip->dw; + + SET_RW(dw, chan->dir, int_clear, + FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id))); +} + +u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir) +{ + return FIELD_GET(EDMA_V0_DONE_INT_MASK, GET_RW(dw, dir, int_status)); +} + +u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) +{ + return FIELD_GET(EDMA_V0_ABORT_INT_MASK, GET_RW(dw, dir, int_status)); +} + +static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) +{ + struct dw_edma_burst *child; + struct dw_edma_v0_lli *lli; + struct dw_edma_v0_llp *llp; + u32 control = 0, i = 0; + u64 sar, dar, addr; + int j; + + lli = (struct dw_edma_v0_lli *)chunk->ll_region.vaddr; + + if (chunk->cb) + control = DW_EDMA_V0_CB; + + j = chunk->bursts_alloc; + list_for_each_entry(child, &chunk->burst->list, list) { + j--; + if (!j) + control |= (DW_EDMA_V0_LIE | DW_EDMA_V0_RIE); + + /* Channel control */ + SET_LL(&lli[i].control, control); + /* Transfer size */ + SET_LL(&lli[i].transfer_size, child->sz); + /* SAR - low, high */ + sar = cpu_to_le64(child->sar); + SET_LL(&lli[i].sar_low, lower_32_bits(sar)); + SET_LL(&lli[i].sar_high, upper_32_bits(sar)); + /* DAR - low, high */ + dar = cpu_to_le64(child->dar); + SET_LL(&lli[i].dar_low, lower_32_bits(dar)); + SET_LL(&lli[i].dar_high, upper_32_bits(dar)); + i++; + } + + llp = (struct dw_edma_v0_llp *)&lli[i]; + control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; + if (!chunk->cb) + control |= DW_EDMA_V0_CB; + + /* Channel control */ + SET_LL(&llp->control, control); + /* Linked list - low, high */ + addr = cpu_to_le64(chunk->ll_region.paddr); + SET_LL(&llp->llp_low, lower_32_bits(addr)); + SET_LL(&llp->llp_high, upper_32_bits(addr)); +} + +void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) +{ + struct dw_edma_chan *chan = chunk->chan; + struct dw_edma *dw = chan->chip->dw; + u32 tmp; + u64 llp; + + dw_edma_v0_core_write_chunk(chunk); + + if (first) { + /* Enable engine */ + SET_RW(dw, chan->dir, engine_en, BIT(0)); + /* Interrupt unmask - done, abort */ + tmp = GET_RW(dw, chan->dir, int_mask); + tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); + tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); + SET_RW(dw, chan->dir, int_mask, tmp); + /* Linked list error */ + tmp = GET_RW(dw, chan->dir, linked_list_err_en); + tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); + SET_RW(dw, chan->dir, linked_list_err_en, tmp); + /* Channel control */ + SET_CH(dw, chan->dir, chan->id, ch_control1, + (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); + /* Linked list - low, high */ + llp = cpu_to_le64(chunk->ll_region.paddr); + SET_CH(dw, chan->dir, chan->id, llp_low, lower_32_bits(llp)); + SET_CH(dw, chan->dir, chan->id, llp_high, upper_32_bits(llp)); + } + /* Doorbell */ + SET_RW(dw, chan->dir, doorbell, + FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); +} + +int dw_edma_v0_core_device_config(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->chip->dw; + u32 tmp = 0; + + /* MSI done addr - low, high */ + SET_RW(dw, chan->dir, done_imwr_low, chan->msi.address_lo); + SET_RW(dw, chan->dir, done_imwr_high, chan->msi.address_hi); + /* MSI abort addr - low, high */ + SET_RW(dw, chan->dir, abort_imwr_low, chan->msi.address_lo); + SET_RW(dw, chan->dir, abort_imwr_high, chan->msi.address_hi); + /* MSI data - low, high */ + switch (chan->id) { + case 0: + case 1: + tmp = GET_RW(dw, chan->dir, ch01_imwr_data); + break; + + case 2: + case 3: + tmp = GET_RW(dw, chan->dir, ch23_imwr_data); + break; + + case 4: + case 5: + tmp = GET_RW(dw, chan->dir, ch45_imwr_data); + break; + + case 6: + case 7: + tmp = GET_RW(dw, chan->dir, ch67_imwr_data); + break; + } + + if (chan->id & BIT(0)) { + /* Channel odd {1, 3, 5, 7} */ + tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK; + tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK, + chan->msi.data); + } else { + /* Channel even {0, 2, 4, 6} */ + tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK; + tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK, + chan->msi.data); + } + + switch (chan->id) { + case 0: + case 1: + SET_RW(dw, chan->dir, ch01_imwr_data, tmp); + break; + + case 2: + case 3: + SET_RW(dw, chan->dir, ch23_imwr_data, tmp); + break; + + case 4: + case 5: + SET_RW(dw, chan->dir, ch45_imwr_data, tmp); + break; + + case 6: + case 7: + SET_RW(dw, chan->dir, ch67_imwr_data, tmp); + break; + } + + return 0; +} + +/* eDMA debugfs callbacks */ +void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip) +{ + dw_edma_v0_debugfs_on(chip); +} + +void dw_edma_v0_core_debugfs_off(void) +{ + dw_edma_v0_debugfs_off(); +} diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.h b/drivers/dma/dw-edma/dw-edma-v0-core.h new file mode 100644 index 000000000000..abae1527f1f9 --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-v0-core.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA v0 core + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#ifndef _DW_EDMA_V0_CORE_H +#define _DW_EDMA_V0_CORE_H + +#include <linux/dma/edma.h> + +/* eDMA management callbacks */ +void dw_edma_v0_core_off(struct dw_edma *chan); +u16 dw_edma_v0_core_ch_count(struct dw_edma *chan, enum dw_edma_dir dir); +enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan); +void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan); +void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan); +u32 dw_edma_v0_core_status_done_int(struct dw_edma *chan, enum dw_edma_dir dir); +u32 dw_edma_v0_core_status_abort_int(struct dw_edma *chan, enum dw_edma_dir dir); +void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first); +int dw_edma_v0_core_device_config(struct dw_edma_chan *chan); +/* eDMA debug fs callbacks */ +void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip); +void dw_edma_v0_core_debugfs_off(void); + +#endif /* _DW_EDMA_V0_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c new file mode 100644 index 000000000000..3226f528cc11 --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA v0 core + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#include <linux/debugfs.h> +#include <linux/bitfield.h> + +#include "dw-edma-v0-debugfs.h" +#include "dw-edma-v0-regs.h" +#include "dw-edma-core.h" + +#define REGS_ADDR(name) \ + ((dma_addr_t *)®s->name) +#define REGISTER(name) \ + { #name, REGS_ADDR(name) } + +#define WR_REGISTER(name) \ + { #name, REGS_ADDR(wr_##name) } +#define RD_REGISTER(name) \ + { #name, REGS_ADDR(rd_##name) } + +#define WR_REGISTER_LEGACY(name) \ + { #name, REGS_ADDR(type.legacy.wr_##name) } +#define RD_REGISTER_LEGACY(name) \ + { #name, REGS_ADDR(type.legacy.rd_##name) } + +#define WR_REGISTER_UNROLL(name) \ + { #name, REGS_ADDR(type.unroll.wr_##name) } +#define RD_REGISTER_UNROLL(name) \ + { #name, REGS_ADDR(type.unroll.rd_##name) } + +#define WRITE_STR "write" +#define READ_STR "read" +#define CHANNEL_STR "channel" +#define REGISTERS_STR "registers" + +static struct dentry *base_dir; +static struct dw_edma *dw; +static struct dw_edma_v0_regs *regs; + +static struct { + void *start; + void *end; +} lim[2][EDMA_V0_MAX_NR_CH]; + +struct debugfs_entries { + char name[24]; + dma_addr_t *reg; +}; + +static int dw_edma_debugfs_u32_get(void *data, u64 *val) +{ + if (dw->mode == EDMA_MODE_LEGACY && + data >= (void *)®s->type.legacy.ch) { + void *ptr = (void *)®s->type.legacy.ch; + u32 viewport_sel = 0; + unsigned long flags; + u16 ch; + + for (ch = 0; ch < dw->wr_ch_cnt; ch++) + if (lim[0][ch].start >= data && data < lim[0][ch].end) { + ptr += (data - lim[0][ch].start); + goto legacy_sel_wr; + } + + for (ch = 0; ch < dw->rd_ch_cnt; ch++) + if (lim[1][ch].start >= data && data < lim[1][ch].end) { + ptr += (data - lim[1][ch].start); + goto legacy_sel_rd; + } + + return 0; +legacy_sel_rd: + viewport_sel = BIT(31); +legacy_sel_wr: + viewport_sel |= FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); + + raw_spin_lock_irqsave(&dw->lock, flags); + + writel(viewport_sel, ®s->type.legacy.viewport_sel); + *val = readl(ptr); + + raw_spin_unlock_irqrestore(&dw->lock, flags); + } else { + *val = readl(data); + } + + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_x32, dw_edma_debugfs_u32_get, NULL, "0x%08llx\n"); + +static void dw_edma_debugfs_create_x32(const struct debugfs_entries entries[], + int nr_entries, struct dentry *dir) +{ + int i; + + for (i = 0; i < nr_entries; i++) { + if (!debugfs_create_file_unsafe(entries[i].name, 0444, dir, + entries[i].reg, &fops_x32)) + break; + } +} + +static void dw_edma_debugfs_regs_ch(struct dw_edma_v0_ch_regs *regs, + struct dentry *dir) +{ + int nr_entries; + const struct debugfs_entries debugfs_regs[] = { + REGISTER(ch_control1), + REGISTER(ch_control2), + REGISTER(transfer_size), + REGISTER(sar_low), + REGISTER(sar_high), + REGISTER(dar_low), + REGISTER(dar_high), + REGISTER(llp_low), + REGISTER(llp_high), + }; + + nr_entries = ARRAY_SIZE(debugfs_regs); + dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, dir); +} + +static void dw_edma_debugfs_regs_wr(struct dentry *dir) +{ + const struct debugfs_entries debugfs_regs[] = { + /* eDMA global registers */ + WR_REGISTER(engine_en), + WR_REGISTER(doorbell), + WR_REGISTER(ch_arb_weight_low), + WR_REGISTER(ch_arb_weight_high), + /* eDMA interrupts registers */ + WR_REGISTER(int_status), + WR_REGISTER(int_mask), + WR_REGISTER(int_clear), + WR_REGISTER(err_status), + WR_REGISTER(done_imwr_low), + WR_REGISTER(done_imwr_high), + WR_REGISTER(abort_imwr_low), + WR_REGISTER(abort_imwr_high), + WR_REGISTER(ch01_imwr_data), + WR_REGISTER(ch23_imwr_data), + WR_REGISTER(ch45_imwr_data), + WR_REGISTER(ch67_imwr_data), + WR_REGISTER(linked_list_err_en), + }; + const struct debugfs_entries debugfs_unroll_regs[] = { + /* eDMA channel context grouping */ + WR_REGISTER_UNROLL(engine_chgroup), + WR_REGISTER_UNROLL(engine_hshake_cnt_low), + WR_REGISTER_UNROLL(engine_hshake_cnt_high), + WR_REGISTER_UNROLL(ch0_pwr_en), + WR_REGISTER_UNROLL(ch1_pwr_en), + WR_REGISTER_UNROLL(ch2_pwr_en), + WR_REGISTER_UNROLL(ch3_pwr_en), + WR_REGISTER_UNROLL(ch4_pwr_en), + WR_REGISTER_UNROLL(ch5_pwr_en), + WR_REGISTER_UNROLL(ch6_pwr_en), + WR_REGISTER_UNROLL(ch7_pwr_en), + }; + struct dentry *regs_dir, *ch_dir; + int nr_entries, i; + char name[16]; + + regs_dir = debugfs_create_dir(WRITE_STR, dir); + if (!regs_dir) + return; + + nr_entries = ARRAY_SIZE(debugfs_regs); + dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); + + if (dw->mode == EDMA_MODE_UNROLL) { + nr_entries = ARRAY_SIZE(debugfs_unroll_regs); + dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries, + regs_dir); + } + + for (i = 0; i < dw->wr_ch_cnt; i++) { + snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i); + + ch_dir = debugfs_create_dir(name, regs_dir); + if (!ch_dir) + return; + + dw_edma_debugfs_regs_ch(®s->type.unroll.ch[i].wr, ch_dir); + + lim[0][i].start = ®s->type.unroll.ch[i].wr; + lim[0][i].end = ®s->type.unroll.ch[i].padding_1[0]; + } +} + +static void dw_edma_debugfs_regs_rd(struct dentry *dir) +{ + const struct debugfs_entries debugfs_regs[] = { + /* eDMA global registers */ + RD_REGISTER(engine_en), + RD_REGISTER(doorbell), + RD_REGISTER(ch_arb_weight_low), + RD_REGISTER(ch_arb_weight_high), + /* eDMA interrupts registers */ + RD_REGISTER(int_status), + RD_REGISTER(int_mask), + RD_REGISTER(int_clear), + RD_REGISTER(err_status_low), + RD_REGISTER(err_status_high), + RD_REGISTER(linked_list_err_en), + RD_REGISTER(done_imwr_low), + RD_REGISTER(done_imwr_high), + RD_REGISTER(abort_imwr_low), + RD_REGISTER(abort_imwr_high), + RD_REGISTER(ch01_imwr_data), + RD_REGISTER(ch23_imwr_data), + RD_REGISTER(ch45_imwr_data), + RD_REGISTER(ch67_imwr_data), + }; + const struct debugfs_entries debugfs_unroll_regs[] = { + /* eDMA channel context grouping */ + RD_REGISTER_UNROLL(engine_chgroup), + RD_REGISTER_UNROLL(engine_hshake_cnt_low), + RD_REGISTER_UNROLL(engine_hshake_cnt_high), + RD_REGISTER_UNROLL(ch0_pwr_en), + RD_REGISTER_UNROLL(ch1_pwr_en), + RD_REGISTER_UNROLL(ch2_pwr_en), + RD_REGISTER_UNROLL(ch3_pwr_en), + RD_REGISTER_UNROLL(ch4_pwr_en), + RD_REGISTER_UNROLL(ch5_pwr_en), + RD_REGISTER_UNROLL(ch6_pwr_en), + RD_REGISTER_UNROLL(ch7_pwr_en), + }; + struct dentry *regs_dir, *ch_dir; + int nr_entries, i; + char name[16]; + + regs_dir = debugfs_create_dir(READ_STR, dir); + if (!regs_dir) + return; + + nr_entries = ARRAY_SIZE(debugfs_regs); + dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); + + if (dw->mode == EDMA_MODE_UNROLL) { + nr_entries = ARRAY_SIZE(debugfs_unroll_regs); + dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries, + regs_dir); + } + + for (i = 0; i < dw->rd_ch_cnt; i++) { + snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i); + + ch_dir = debugfs_create_dir(name, regs_dir); + if (!ch_dir) + return; + + dw_edma_debugfs_regs_ch(®s->type.unroll.ch[i].rd, ch_dir); + + lim[1][i].start = ®s->type.unroll.ch[i].rd; + lim[1][i].end = ®s->type.unroll.ch[i].padding_2[0]; + } +} + +static void dw_edma_debugfs_regs(void) +{ + const struct debugfs_entries debugfs_regs[] = { + REGISTER(ctrl_data_arb_prior), + REGISTER(ctrl), + }; + struct dentry *regs_dir; + int nr_entries; + + regs_dir = debugfs_create_dir(REGISTERS_STR, base_dir); + if (!regs_dir) + return; + + nr_entries = ARRAY_SIZE(debugfs_regs); + dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); + + dw_edma_debugfs_regs_wr(regs_dir); + dw_edma_debugfs_regs_rd(regs_dir); +} + +void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip) +{ + dw = chip->dw; + if (!dw) + return; + + regs = (struct dw_edma_v0_regs *)dw->rg_region.vaddr; + if (!regs) + return; + + base_dir = debugfs_create_dir(dw->name, 0); + if (!base_dir) + return; + + debugfs_create_u32("version", 0444, base_dir, &dw->version); + debugfs_create_u32("mode", 0444, base_dir, &dw->mode); + debugfs_create_u16("wr_ch_cnt", 0444, base_dir, &dw->wr_ch_cnt); + debugfs_create_u16("rd_ch_cnt", 0444, base_dir, &dw->rd_ch_cnt); + + dw_edma_debugfs_regs(); +} + +void dw_edma_v0_debugfs_off(void) +{ + debugfs_remove_recursive(base_dir); +} diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.h b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h new file mode 100644 index 000000000000..5450a0a94193 --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA v0 core + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#ifndef _DW_EDMA_V0_DEBUG_FS_H +#define _DW_EDMA_V0_DEBUG_FS_H + +#include <linux/dma/edma.h> + +#ifdef CONFIG_DEBUG_FS +void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip); +void dw_edma_v0_debugfs_off(void); +#else +static inline void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip) +{ +} + +static inline void dw_edma_v0_debugfs_off(void) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#endif /* _DW_EDMA_V0_DEBUG_FS_H */ diff --git a/drivers/dma/dw-edma/dw-edma-v0-regs.h b/drivers/dma/dw-edma/dw-edma-v0-regs.h new file mode 100644 index 000000000000..cd6476884507 --- /dev/null +++ b/drivers/dma/dw-edma/dw-edma-v0-regs.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA v0 core + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#ifndef _DW_EDMA_V0_REGS_H +#define _DW_EDMA_V0_REGS_H + +#include <linux/dmaengine.h> + +#define EDMA_V0_MAX_NR_CH 8 +#define EDMA_V0_VIEWPORT_MASK GENMASK(2, 0) +#define EDMA_V0_DONE_INT_MASK GENMASK(7, 0) +#define EDMA_V0_ABORT_INT_MASK GENMASK(23, 16) +#define EDMA_V0_WRITE_CH_COUNT_MASK GENMASK(3, 0) +#define EDMA_V0_READ_CH_COUNT_MASK GENMASK(19, 16) +#define EDMA_V0_CH_STATUS_MASK GENMASK(6, 5) +#define EDMA_V0_DOORBELL_CH_MASK GENMASK(2, 0) +#define EDMA_V0_LINKED_LIST_ERR_MASK GENMASK(7, 0) + +#define EDMA_V0_CH_ODD_MSI_DATA_MASK GENMASK(31, 16) +#define EDMA_V0_CH_EVEN_MSI_DATA_MASK GENMASK(15, 0) + +struct dw_edma_v0_ch_regs { + u32 ch_control1; /* 0x000 */ + u32 ch_control2; /* 0x004 */ + u32 transfer_size; /* 0x008 */ + u32 sar_low; /* 0x00c */ + u32 sar_high; /* 0x010 */ + u32 dar_low; /* 0x014 */ + u32 dar_high; /* 0x018 */ + u32 llp_low; /* 0x01c */ + u32 llp_high; /* 0x020 */ +}; + +struct dw_edma_v0_ch { + struct dw_edma_v0_ch_regs wr; /* 0x200 */ + u32 padding_1[55]; /* [0x224..0x2fc] */ + struct dw_edma_v0_ch_regs rd; /* 0x300 */ + u32 padding_2[55]; /* [0x224..0x2fc] */ +}; + +struct dw_edma_v0_unroll { + u32 padding_1; /* 0x0f8 */ + u32 wr_engine_chgroup; /* 0x100 */ + u32 rd_engine_chgroup; /* 0x104 */ + u32 wr_engine_hshake_cnt_low; /* 0x108 */ + u32 wr_engine_hshake_cnt_high; /* 0x10c */ + u32 padding_2[2]; /* [0x110..0x114] */ + u32 rd_engine_hshake_cnt_low; /* 0x118 */ + u32 rd_engine_hshake_cnt_high; /* 0x11c */ + u32 padding_3[2]; /* [0x120..0x124] */ + u32 wr_ch0_pwr_en; /* 0x128 */ + u32 wr_ch1_pwr_en; /* 0x12c */ + u32 wr_ch2_pwr_en; /* 0x130 */ + u32 wr_ch3_pwr_en; /* 0x134 */ + u32 wr_ch4_pwr_en; /* 0x138 */ + u32 wr_ch5_pwr_en; /* 0x13c */ + u32 wr_ch6_pwr_en; /* 0x140 */ + u32 wr_ch7_pwr_en; /* 0x144 */ + u32 padding_4[8]; /* [0x148..0x164] */ + u32 rd_ch0_pwr_en; /* 0x168 */ + u32 rd_ch1_pwr_en; /* 0x16c */ + u32 rd_ch2_pwr_en; /* 0x170 */ + u32 rd_ch3_pwr_en; /* 0x174 */ + u32 rd_ch4_pwr_en; /* 0x178 */ + u32 rd_ch5_pwr_en; /* 0x18c */ + u32 rd_ch6_pwr_en; /* 0x180 */ + u32 rd_ch7_pwr_en; /* 0x184 */ + u32 padding_5[30]; /* [0x188..0x1fc] */ + struct dw_edma_v0_ch ch[EDMA_V0_MAX_NR_CH]; /* [0x200..0x1120] */ +}; + +struct dw_edma_v0_legacy { + u32 viewport_sel; /* 0x0f8 */ + struct dw_edma_v0_ch_regs ch; /* [0x100..0x120] */ +}; + +struct dw_edma_v0_regs { + /* eDMA global registers */ + u32 ctrl_data_arb_prior; /* 0x000 */ + u32 padding_1; /* 0x004 */ + u32 ctrl; /* 0x008 */ + u32 wr_engine_en; /* 0x00c */ + u32 wr_doorbell; /* 0x010 */ + u32 padding_2; /* 0x014 */ + u32 wr_ch_arb_weight_low; /* 0x018 */ + u32 wr_ch_arb_weight_high; /* 0x01c */ + u32 padding_3[3]; /* [0x020..0x028] */ + u32 rd_engine_en; /* 0x02c */ + u32 rd_doorbell; /* 0x030 */ + u32 padding_4; /* 0x034 */ + u32 rd_ch_arb_weight_low; /* 0x038 */ + u32 rd_ch_arb_weight_high; /* 0x03c */ + u32 padding_5[3]; /* [0x040..0x048] */ + /* eDMA interrupts registers */ + u32 wr_int_status; /* 0x04c */ + u32 padding_6; /* 0x050 */ + u32 wr_int_mask; /* 0x054 */ + u32 wr_int_clear; /* 0x058 */ + u32 wr_err_status; /* 0x05c */ + u32 wr_done_imwr_low; /* 0x060 */ + u32 wr_done_imwr_high; /* 0x064 */ + u32 wr_abort_imwr_low; /* 0x068 */ + u32 wr_abort_imwr_high; /* 0x06c */ + u32 wr_ch01_imwr_data; /* 0x070 */ + u32 wr_ch23_imwr_data; /* 0x074 */ + u32 wr_ch45_imwr_data; /* 0x078 */ + u32 wr_ch67_imwr_data; /* 0x07c */ + u32 padding_7[4]; /* [0x080..0x08c] */ + u32 wr_linked_list_err_en; /* 0x090 */ + u32 padding_8[3]; /* [0x094..0x09c] */ + u32 rd_int_status; /* 0x0a0 */ + u32 padding_9; /* 0x0a4 */ + u32 rd_int_mask; /* 0x0a8 */ + u32 rd_int_clear; /* 0x0ac */ + u32 padding_10; /* 0x0b0 */ + u32 rd_err_status_low; /* 0x0b4 */ + u32 rd_err_status_high; /* 0x0b8 */ + u32 padding_11[2]; /* [0x0bc..0x0c0] */ + u32 rd_linked_list_err_en; /* 0x0c4 */ + u32 padding_12; /* 0x0c8 */ + u32 rd_done_imwr_low; /* 0x0cc */ + u32 rd_done_imwr_high; /* 0x0d0 */ + u32 rd_abort_imwr_low; /* 0x0d4 */ + u32 rd_abort_imwr_high; /* 0x0d8 */ + u32 rd_ch01_imwr_data; /* 0x0dc */ + u32 rd_ch23_imwr_data; /* 0x0e0 */ + u32 rd_ch45_imwr_data; /* 0x0e4 */ + u32 rd_ch67_imwr_data; /* 0x0e8 */ + u32 padding_13[4]; /* [0x0ec..0x0f8] */ + /* eDMA channel context grouping */ + union dw_edma_v0_type { + struct dw_edma_v0_legacy legacy; /* [0x0f8..0x120] */ + struct dw_edma_v0_unroll unroll; /* [0x0f8..0x1120] */ + } type; +}; + +struct dw_edma_v0_lli { + u32 control; + u32 transfer_size; + u32 sar_low; + u32 sar_high; + u32 dar_low; + u32 dar_high; +}; + +struct dw_edma_v0_llp { + u32 control; + u32 reserved; + u32 llp_low; + u32 llp_high; +}; + +#endif /* _DW_EDMA_V0_REGS_H */ diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c index e79a75db0852..8de87b15a988 100644 --- a/drivers/dma/dw/pci.c +++ b/drivers/dma/dw/pci.c @@ -15,10 +15,13 @@ struct dw_dma_pci_data { const struct dw_dma_platform_data *pdata; int (*probe)(struct dw_dma_chip *chip); + int (*remove)(struct dw_dma_chip *chip); + struct dw_dma_chip *chip; }; static const struct dw_dma_pci_data dw_pci_data = { .probe = dw_dma_probe, + .remove = dw_dma_remove, }; static const struct dw_dma_platform_data idma32_pdata = { @@ -34,11 +37,13 @@ static const struct dw_dma_platform_data idma32_pdata = { static const struct dw_dma_pci_data idma32_pci_data = { .pdata = &idma32_pdata, .probe = idma32_dma_probe, + .remove = idma32_dma_remove, }; static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { - const struct dw_dma_pci_data *data = (void *)pid->driver_data; + const struct dw_dma_pci_data *drv_data = (void *)pid->driver_data; + struct dw_dma_pci_data *data; struct dw_dma_chip *chip; int ret; @@ -63,6 +68,10 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) if (ret) return ret; + data = devm_kmemdup(&pdev->dev, drv_data, sizeof(*drv_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL); if (!chip) return -ENOMEM; @@ -73,21 +82,24 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) chip->irq = pdev->irq; chip->pdata = data->pdata; + data->chip = chip; + ret = data->probe(chip); if (ret) return ret; - pci_set_drvdata(pdev, chip); + pci_set_drvdata(pdev, data); return 0; } static void dw_pci_remove(struct pci_dev *pdev) { - struct dw_dma_chip *chip = pci_get_drvdata(pdev); + struct dw_dma_pci_data *data = pci_get_drvdata(pdev); + struct dw_dma_chip *chip = data->chip; int ret; - ret = dw_dma_remove(chip); + ret = data->remove(chip); if (ret) dev_warn(&pdev->dev, "can't remove device properly: %d\n", ret); } @@ -96,16 +108,16 @@ static void dw_pci_remove(struct pci_dev *pdev) static int dw_pci_suspend_late(struct device *dev) { - struct pci_dev *pci = to_pci_dev(dev); - struct dw_dma_chip *chip = pci_get_drvdata(pci); + struct dw_dma_pci_data *data = dev_get_drvdata(dev); + struct dw_dma_chip *chip = data->chip; return do_dw_dma_disable(chip); }; static int dw_pci_resume_early(struct device *dev) { - struct pci_dev *pci = to_pci_dev(dev); - struct dw_dma_chip *chip = pci_get_drvdata(pci); + struct dw_dma_pci_data *data = dev_get_drvdata(dev); + struct dw_dma_chip *chip = data->chip; return do_dw_dma_enable(chip); }; @@ -131,6 +143,11 @@ static const struct pci_device_id dw_pci_id_table[] = { { PCI_VDEVICE(INTEL, 0x2286), (kernel_ulong_t)&dw_pci_data }, { PCI_VDEVICE(INTEL, 0x22c0), (kernel_ulong_t)&dw_pci_data }, + /* Elkhart Lake iDMA 32-bit (OSE DMA) */ + { PCI_VDEVICE(INTEL, 0x4bb4), (kernel_ulong_t)&idma32_pci_data }, + { PCI_VDEVICE(INTEL, 0x4bb5), (kernel_ulong_t)&idma32_pci_data }, + { PCI_VDEVICE(INTEL, 0x4bb6), (kernel_ulong_t)&idma32_pci_data }, + /* Haswell */ { PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_pci_data }, diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index 680b2a00a953..44d92c34dec3 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -47,7 +47,7 @@ static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) struct edma_regs *regs = &fsl_chan->edma->regs; u32 ch = fsl_chan->vchan.chan.chan_id; - if (fsl_chan->edma->version == v1) { + if (fsl_chan->edma->drvdata->version == v1) { edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), regs->seei); edma_writeb(fsl_chan->edma, ch, regs->serq); } else { @@ -64,7 +64,7 @@ void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) struct edma_regs *regs = &fsl_chan->edma->regs; u32 ch = fsl_chan->vchan.chan.chan_id; - if (fsl_chan->edma->version == v1) { + if (fsl_chan->edma->drvdata->version == v1) { edma_writeb(fsl_chan->edma, ch, regs->cerq); edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), regs->ceei); } else { @@ -77,22 +77,33 @@ void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) } EXPORT_SYMBOL_GPL(fsl_edma_disable_request); +static void mux_configure8(struct fsl_edma_chan *fsl_chan, void __iomem *addr, + u32 off, u32 slot, bool enable) +{ + u8 val8; + + if (enable) + val8 = EDMAMUX_CHCFG_ENBL | slot; + else + val8 = EDMAMUX_CHCFG_DIS; + + iowrite8(val8, addr + off); +} + void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, unsigned int slot, bool enable) { u32 ch = fsl_chan->vchan.chan.chan_id; void __iomem *muxaddr; unsigned int chans_per_mux, ch_off; + u32 dmamux_nr = fsl_chan->edma->drvdata->dmamuxs; - chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; + chans_per_mux = fsl_chan->edma->n_chans / dmamux_nr; ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; slot = EDMAMUX_CHCFG_SOURCE(slot); - if (enable) - iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); - else - iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); + mux_configure8(fsl_chan, muxaddr, ch_off, slot, enable); } EXPORT_SYMBOL_GPL(fsl_edma_chan_mux); @@ -647,28 +658,28 @@ void fsl_edma_setup_regs(struct fsl_edma_engine *edma) edma->regs.erql = edma->membase + EDMA_ERQ; edma->regs.eeil = edma->membase + EDMA_EEI; - edma->regs.serq = edma->membase + ((edma->version == v1) ? - EDMA_SERQ : EDMA64_SERQ); - edma->regs.cerq = edma->membase + ((edma->version == v1) ? - EDMA_CERQ : EDMA64_CERQ); - edma->regs.seei = edma->membase + ((edma->version == v1) ? - EDMA_SEEI : EDMA64_SEEI); - edma->regs.ceei = edma->membase + ((edma->version == v1) ? - EDMA_CEEI : EDMA64_CEEI); - edma->regs.cint = edma->membase + ((edma->version == v1) ? - EDMA_CINT : EDMA64_CINT); - edma->regs.cerr = edma->membase + ((edma->version == v1) ? - EDMA_CERR : EDMA64_CERR); - edma->regs.ssrt = edma->membase + ((edma->version == v1) ? - EDMA_SSRT : EDMA64_SSRT); - edma->regs.cdne = edma->membase + ((edma->version == v1) ? - EDMA_CDNE : EDMA64_CDNE); - edma->regs.intl = edma->membase + ((edma->version == v1) ? - EDMA_INTR : EDMA64_INTL); - edma->regs.errl = edma->membase + ((edma->version == v1) ? - EDMA_ERR : EDMA64_ERRL); - - if (edma->version == v2) { + edma->regs.serq = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_SERQ : EDMA_SERQ); + edma->regs.cerq = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_CERQ : EDMA_CERQ); + edma->regs.seei = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_SEEI : EDMA_SEEI); + edma->regs.ceei = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_CEEI : EDMA_CEEI); + edma->regs.cint = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_CINT : EDMA_CINT); + edma->regs.cerr = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_CERR : EDMA_CERR); + edma->regs.ssrt = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_SSRT : EDMA_SSRT); + edma->regs.cdne = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_CDNE : EDMA_CDNE); + edma->regs.intl = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_INTL : EDMA_INTR); + edma->regs.errl = edma->membase + ((edma->drvdata->version == v2) ? + EDMA64_ERRL : EDMA_ERR); + + if (edma->drvdata->version == v2) { edma->regs.erqh = edma->membase + EDMA64_ERQH; edma->regs.eeih = edma->membase + EDMA64_EEIH; edma->regs.errh = edma->membase + EDMA64_ERRH; diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index c53f76eeb4d3..4e175560292c 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -7,6 +7,7 @@ #define _FSL_EDMA_COMMON_H_ #include <linux/dma-direction.h> +#include <linux/platform_device.h> #include "virt-dma.h" #define EDMA_CR_EDBG BIT(1) @@ -140,17 +141,24 @@ enum edma_version { v2, /* 64ch Coldfire */ }; +struct fsl_edma_drvdata { + enum edma_version version; + u32 dmamuxs; + int (*setup_irq)(struct platform_device *pdev, + struct fsl_edma_engine *fsl_edma); +}; + struct fsl_edma_engine { struct dma_device dma_dev; void __iomem *membase; void __iomem *muxbase[DMAMUX_NR]; struct clk *muxclk[DMAMUX_NR]; struct mutex fsl_edma_mutex; + const struct fsl_edma_drvdata *drvdata; u32 n_chans; int txirq; int errirq; bool big_endian; - enum edma_version version; struct edma_regs regs; struct fsl_edma_chan chans[]; }; diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index 0ddad3adb761..fcbad6ae954a 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -92,7 +92,8 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; struct dma_chan *chan, *_chan; struct fsl_edma_chan *fsl_chan; - unsigned long chans_per_mux = fsl_edma->n_chans / DMAMUX_NR; + u32 dmamux_nr = fsl_edma->drvdata->dmamuxs; + unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr; if (dma_spec->args_count != 2) return NULL; @@ -180,16 +181,38 @@ static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks) clk_disable_unprepare(fsl_edma->muxclk[i]); } +static struct fsl_edma_drvdata vf610_data = { + .version = v1, + .dmamuxs = DMAMUX_NR, + .setup_irq = fsl_edma_irq_init, +}; + +static const struct of_device_id fsl_edma_dt_ids[] = { + { .compatible = "fsl,vf610-edma", .data = &vf610_data}, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); + static int fsl_edma_probe(struct platform_device *pdev) { + const struct of_device_id *of_id = + of_match_device(fsl_edma_dt_ids, &pdev->dev); struct device_node *np = pdev->dev.of_node; struct fsl_edma_engine *fsl_edma; + const struct fsl_edma_drvdata *drvdata = NULL; struct fsl_edma_chan *fsl_chan; struct edma_regs *regs; struct resource *res; int len, chans; int ret, i; + if (of_id) + drvdata = of_id->data; + if (!drvdata) { + dev_err(&pdev->dev, "unable to find driver data\n"); + return -EINVAL; + } + ret = of_property_read_u32(np, "dma-channels", &chans); if (ret) { dev_err(&pdev->dev, "Can't get dma-channels.\n"); @@ -201,7 +224,7 @@ static int fsl_edma_probe(struct platform_device *pdev) if (!fsl_edma) return -ENOMEM; - fsl_edma->version = v1; + fsl_edma->drvdata = drvdata; fsl_edma->n_chans = chans; mutex_init(&fsl_edma->fsl_edma_mutex); @@ -213,7 +236,7 @@ static int fsl_edma_probe(struct platform_device *pdev) fsl_edma_setup_regs(fsl_edma); regs = &fsl_edma->regs; - for (i = 0; i < DMAMUX_NR; i++) { + for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) { char clkname[32]; res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); @@ -259,7 +282,7 @@ static int fsl_edma_probe(struct platform_device *pdev) } edma_writel(fsl_edma, ~0, regs->intl); - ret = fsl_edma_irq_init(pdev, fsl_edma); + ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma); if (ret) return ret; @@ -291,7 +314,7 @@ static int fsl_edma_probe(struct platform_device *pdev) if (ret) { dev_err(&pdev->dev, "Can't register Freescale eDMA engine. (%d)\n", ret); - fsl_disable_clocks(fsl_edma, DMAMUX_NR); + fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); return ret; } @@ -300,7 +323,7 @@ static int fsl_edma_probe(struct platform_device *pdev) dev_err(&pdev->dev, "Can't register Freescale eDMA of_dma. (%d)\n", ret); dma_async_device_unregister(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, DMAMUX_NR); + fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); return ret; } @@ -319,7 +342,7 @@ static int fsl_edma_remove(struct platform_device *pdev) fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); of_dma_controller_free(np); dma_async_device_unregister(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, DMAMUX_NR); + fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); return 0; } @@ -378,12 +401,6 @@ static const struct dev_pm_ops fsl_edma_pm_ops = { .resume_early = fsl_edma_resume_early, }; -static const struct of_device_id fsl_edma_dt_ids[] = { - { .compatible = "fsl,vf610-edma", }, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); - static struct platform_driver fsl_edma_driver = { .driver = { .name = "fsl-edma", diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c index 60b062c3647b..8e341c0c13bc 100644 --- a/drivers/dma/fsl-qdma.c +++ b/drivers/dma/fsl-qdma.c @@ -113,6 +113,7 @@ /* Field definition for Descriptor offset */ #define QDMA_CCDF_STATUS 20 #define QDMA_CCDF_OFFSET 20 +#define QDMA_SDDF_CMD(x) (((u64)(x)) << 32) /* Field definition for safe loop count*/ #define FSL_QDMA_HALT_COUNT 1500 @@ -341,6 +342,7 @@ static void fsl_qdma_free_chan_resources(struct dma_chan *chan) static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, dma_addr_t dst, dma_addr_t src, u32 len) { + u32 cmd; struct fsl_qdma_format *sdf, *ddf; struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest; @@ -369,14 +371,14 @@ static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, /* This entry is the last entry. */ qdma_csgf_set_f(csgf_dest, len); /* Descriptor Buffer */ - sdf->data = - cpu_to_le64(FSL_QDMA_CMD_RWTTYPE << - FSL_QDMA_CMD_RWTTYPE_OFFSET); - ddf->data = - cpu_to_le64(FSL_QDMA_CMD_RWTTYPE << - FSL_QDMA_CMD_RWTTYPE_OFFSET); - ddf->data |= - cpu_to_le64(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET); + cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE << + FSL_QDMA_CMD_RWTTYPE_OFFSET); + sdf->data = QDMA_SDDF_CMD(cmd); + + cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE << + FSL_QDMA_CMD_RWTTYPE_OFFSET); + cmd |= cpu_to_le32(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET); + ddf->data = QDMA_SDDF_CMD(cmd); } /* diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c index 0c2610066ba9..025d8ad5a63c 100644 --- a/drivers/dma/hsu/hsu.c +++ b/drivers/dma/hsu/hsu.c @@ -61,10 +61,10 @@ static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc) if (hsuc->direction == DMA_MEM_TO_DEV) { bsr = config->dst_maxburst; - mtsr = config->src_addr_width; + mtsr = config->dst_addr_width; } else if (hsuc->direction == DMA_DEV_TO_MEM) { bsr = config->src_maxburst; - mtsr = config->dst_addr_width; + mtsr = config->src_addr_width; } hsu_chan_disable(hsuc); diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 4ec84a633bd3..a01f4b5d793c 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1934,16 +1934,11 @@ disable_clk_ipg: static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param) { struct sdma_channel *sdmac = to_sdma_chan(chan); - struct sdma_engine *sdma = sdmac->sdma; struct imx_dma_data *data = fn_param; if (!imx_dma_is_general_purpose(chan)) return false; - /* return false if it's not the right device */ - if (sdma->dev->of_node != data->of_node) - return false; - sdmac->data = *data; chan->private = &sdmac->data; @@ -1971,9 +1966,9 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, * be set to sdmac->event_id1. */ data.dma_request2 = 0; - data.of_node = ofdma->of_node; - return dma_request_channel(mask, sdma_filter_fn, &data); + return __dma_request_channel(&mask, sdma_filter_fn, &data, + ofdma->of_node); } static int sdma_probe(struct platform_device *pdev) diff --git a/drivers/dma/mcf-edma.c b/drivers/dma/mcf-edma.c index 7de54b2fafdb..e15bd15a9ef6 100644 --- a/drivers/dma/mcf-edma.c +++ b/drivers/dma/mcf-edma.c @@ -164,6 +164,11 @@ static void mcf_edma_irq_free(struct platform_device *pdev, free_irq(irq, mcf_edma); } +static struct fsl_edma_drvdata mcf_data = { + .version = v2, + .setup_irq = mcf_edma_irq_init, +}; + static int mcf_edma_probe(struct platform_device *pdev) { struct mcf_edma_platform_data *pdata; @@ -187,8 +192,8 @@ static int mcf_edma_probe(struct platform_device *pdev) mcf_edma->n_chans = chans; - /* Set up version for ColdFire edma */ - mcf_edma->version = v2; + /* Set up drvdata for ColdFire edma */ + mcf_edma->drvdata = &mcf_data; mcf_edma->big_endian = 1; if (!mcf_edma->n_chans) { @@ -223,7 +228,7 @@ static int mcf_edma_probe(struct platform_device *pdev) iowrite32(~0, regs->inth); iowrite32(~0, regs->intl); - ret = mcf_edma_irq_init(pdev, mcf_edma); + ret = mcf_edma->drvdata->setup_irq(pdev, mcf_edma); if (ret) return ret; diff --git a/drivers/dma/mediatek/Kconfig b/drivers/dma/mediatek/Kconfig index 7411eb3d419e..1ad63ddc292d 100644 --- a/drivers/dma/mediatek/Kconfig +++ b/drivers/dma/mediatek/Kconfig @@ -25,3 +25,14 @@ config MTK_CQDMA This controller provides the channels which is dedicated to memory-to-memory transfer to offload from CPU. + +config MTK_UART_APDMA + tristate "MediaTek SoCs APDMA support for UART" + depends on OF && SERIAL_8250_MT6577 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the UART DMA engine found on MediaTek MTK SoCs. + When SERIAL_8250_MT6577 is enabled, and if you want to use DMA, + you can enable the config. The DMA engine can only be used + with MediaTek SoCs. diff --git a/drivers/dma/mediatek/Makefile b/drivers/dma/mediatek/Makefile index 13b144594510..5ba39a5edc13 100644 --- a/drivers/dma/mediatek/Makefile +++ b/drivers/dma/mediatek/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_MTK_UART_APDMA) += mtk-uart-apdma.o obj-$(CONFIG_MTK_HSDMA) += mtk-hsdma.o obj-$(CONFIG_MTK_CQDMA) += mtk-cqdma.o diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c new file mode 100644 index 000000000000..546995c20876 --- /dev/null +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -0,0 +1,666 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MediaTek UART APDMA driver. + * + * Copyright (c) 2019 MediaTek Inc. + * Author: Long Cheng <long.cheng@mediatek.com> + */ + +#include <linux/clk.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#include "../virt-dma.h" + +/* The default number of virtual channel */ +#define MTK_UART_APDMA_NR_VCHANS 8 + +#define VFF_EN_B BIT(0) +#define VFF_STOP_B BIT(0) +#define VFF_FLUSH_B BIT(0) +#define VFF_4G_EN_B BIT(0) +/* rx valid size >= vff thre */ +#define VFF_RX_INT_EN_B (BIT(0) | BIT(1)) +/* tx left size >= vff thre */ +#define VFF_TX_INT_EN_B BIT(0) +#define VFF_WARM_RST_B BIT(0) +#define VFF_RX_INT_CLR_B (BIT(0) | BIT(1)) +#define VFF_TX_INT_CLR_B 0 +#define VFF_STOP_CLR_B 0 +#define VFF_EN_CLR_B 0 +#define VFF_INT_EN_CLR_B 0 +#define VFF_4G_SUPPORT_CLR_B 0 + +/* + * interrupt trigger level for tx + * if threshold is n, no polling is required to start tx. + * otherwise need polling VFF_FLUSH. + */ +#define VFF_TX_THRE(n) (n) +/* interrupt trigger level for rx */ +#define VFF_RX_THRE(n) ((n) * 3 / 4) + +#define VFF_RING_SIZE 0xffff +/* invert this bit when wrap ring head again */ +#define VFF_RING_WRAP 0x10000 + +#define VFF_INT_FLAG 0x00 +#define VFF_INT_EN 0x04 +#define VFF_EN 0x08 +#define VFF_RST 0x0c +#define VFF_STOP 0x10 +#define VFF_FLUSH 0x14 +#define VFF_ADDR 0x1c +#define VFF_LEN 0x24 +#define VFF_THRE 0x28 +#define VFF_WPT 0x2c +#define VFF_RPT 0x30 +/* TX: the buffer size HW can read. RX: the buffer size SW can read. */ +#define VFF_VALID_SIZE 0x3c +/* TX: the buffer size SW can write. RX: the buffer size HW can write. */ +#define VFF_LEFT_SIZE 0x40 +#define VFF_DEBUG_STATUS 0x50 +#define VFF_4G_SUPPORT 0x54 + +struct mtk_uart_apdmadev { + struct dma_device ddev; + struct clk *clk; + bool support_33bits; + unsigned int dma_requests; +}; + +struct mtk_uart_apdma_desc { + struct virt_dma_desc vd; + + dma_addr_t addr; + unsigned int avail_len; +}; + +struct mtk_chan { + struct virt_dma_chan vc; + struct dma_slave_config cfg; + struct mtk_uart_apdma_desc *desc; + enum dma_transfer_direction dir; + + void __iomem *base; + unsigned int irq; + + unsigned int rx_status; +}; + +static inline struct mtk_uart_apdmadev * +to_mtk_uart_apdma_dev(struct dma_device *d) +{ + return container_of(d, struct mtk_uart_apdmadev, ddev); +} + +static inline struct mtk_chan *to_mtk_uart_apdma_chan(struct dma_chan *c) +{ + return container_of(c, struct mtk_chan, vc.chan); +} + +static inline struct mtk_uart_apdma_desc *to_mtk_uart_apdma_desc + (struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct mtk_uart_apdma_desc, vd.tx); +} + +static void mtk_uart_apdma_write(struct mtk_chan *c, + unsigned int reg, unsigned int val) +{ + writel(val, c->base + reg); +} + +static unsigned int mtk_uart_apdma_read(struct mtk_chan *c, unsigned int reg) +{ + return readl(c->base + reg); +} + +static void mtk_uart_apdma_desc_free(struct virt_dma_desc *vd) +{ + struct dma_chan *chan = vd->tx.chan; + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + + kfree(c->desc); +} + +static void mtk_uart_apdma_start_tx(struct mtk_chan *c) +{ + struct mtk_uart_apdmadev *mtkd = + to_mtk_uart_apdma_dev(c->vc.chan.device); + struct mtk_uart_apdma_desc *d = c->desc; + unsigned int wpt, vff_sz; + + vff_sz = c->cfg.dst_port_window_size; + if (!mtk_uart_apdma_read(c, VFF_LEN)) { + mtk_uart_apdma_write(c, VFF_ADDR, d->addr); + mtk_uart_apdma_write(c, VFF_LEN, vff_sz); + mtk_uart_apdma_write(c, VFF_THRE, VFF_TX_THRE(vff_sz)); + mtk_uart_apdma_write(c, VFF_WPT, 0); + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); + + if (mtkd->support_33bits) + mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + } + + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); + if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B) + dev_err(c->vc.chan.device->dev, "Enable TX fail\n"); + + if (!mtk_uart_apdma_read(c, VFF_LEFT_SIZE)) { + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B); + return; + } + + wpt = mtk_uart_apdma_read(c, VFF_WPT); + + wpt += c->desc->avail_len; + if ((wpt & VFF_RING_SIZE) == vff_sz) + wpt = (wpt & VFF_RING_WRAP) ^ VFF_RING_WRAP; + + /* Let DMA start moving data */ + mtk_uart_apdma_write(c, VFF_WPT, wpt); + + /* HW auto set to 0 when left size >= threshold */ + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_TX_INT_EN_B); + if (!mtk_uart_apdma_read(c, VFF_FLUSH)) + mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B); +} + +static void mtk_uart_apdma_start_rx(struct mtk_chan *c) +{ + struct mtk_uart_apdmadev *mtkd = + to_mtk_uart_apdma_dev(c->vc.chan.device); + struct mtk_uart_apdma_desc *d = c->desc; + unsigned int vff_sz; + + vff_sz = c->cfg.src_port_window_size; + if (!mtk_uart_apdma_read(c, VFF_LEN)) { + mtk_uart_apdma_write(c, VFF_ADDR, d->addr); + mtk_uart_apdma_write(c, VFF_LEN, vff_sz); + mtk_uart_apdma_write(c, VFF_THRE, VFF_RX_THRE(vff_sz)); + mtk_uart_apdma_write(c, VFF_RPT, 0); + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); + + if (mtkd->support_33bits) + mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + } + + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B); + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); + if (mtk_uart_apdma_read(c, VFF_EN) != VFF_EN_B) + dev_err(c->vc.chan.device->dev, "Enable RX fail\n"); +} + +static void mtk_uart_apdma_tx_handler(struct mtk_chan *c) +{ + struct mtk_uart_apdma_desc *d = c->desc; + + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); + + list_del(&d->vd.node); + vchan_cookie_complete(&d->vd); +} + +static void mtk_uart_apdma_rx_handler(struct mtk_chan *c) +{ + struct mtk_uart_apdma_desc *d = c->desc; + unsigned int len, wg, rg; + int cnt; + + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); + + if (!mtk_uart_apdma_read(c, VFF_VALID_SIZE)) + return; + + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + + len = c->cfg.src_port_window_size; + rg = mtk_uart_apdma_read(c, VFF_RPT); + wg = mtk_uart_apdma_read(c, VFF_WPT); + cnt = (wg & VFF_RING_SIZE) - (rg & VFF_RING_SIZE); + + /* + * The buffer is ring buffer. If wrap bit different, + * represents the start of the next cycle for WPT + */ + if ((rg ^ wg) & VFF_RING_WRAP) + cnt += len; + + c->rx_status = d->avail_len - cnt; + mtk_uart_apdma_write(c, VFF_RPT, wg); + + list_del(&d->vd.node); + vchan_cookie_complete(&d->vd); +} + +static irqreturn_t mtk_uart_apdma_irq_handler(int irq, void *dev_id) +{ + struct dma_chan *chan = (struct dma_chan *)dev_id; + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + if (c->dir == DMA_DEV_TO_MEM) + mtk_uart_apdma_rx_handler(c); + else if (c->dir == DMA_MEM_TO_DEV) + mtk_uart_apdma_tx_handler(c); + spin_unlock_irqrestore(&c->vc.lock, flags); + + return IRQ_HANDLED; +} + +static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) +{ + struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device); + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned int status; + int ret; + + ret = pm_runtime_get_sync(mtkd->ddev.dev); + if (ret < 0) { + pm_runtime_put_noidle(chan->device->dev); + return ret; + } + + mtk_uart_apdma_write(c, VFF_ADDR, 0); + mtk_uart_apdma_write(c, VFF_THRE, 0); + mtk_uart_apdma_write(c, VFF_LEN, 0); + mtk_uart_apdma_write(c, VFF_RST, VFF_WARM_RST_B); + + ret = readx_poll_timeout(readl, c->base + VFF_EN, + status, !status, 10, 100); + if (ret) + return ret; + + ret = request_irq(c->irq, mtk_uart_apdma_irq_handler, + IRQF_TRIGGER_NONE, KBUILD_MODNAME, chan); + if (ret < 0) { + dev_err(chan->device->dev, "Can't request dma IRQ\n"); + return -EINVAL; + } + + if (mtkd->support_33bits) + mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B); + + return ret; +} + +static void mtk_uart_apdma_free_chan_resources(struct dma_chan *chan) +{ + struct mtk_uart_apdmadev *mtkd = to_mtk_uart_apdma_dev(chan->device); + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + + free_irq(c->irq, chan); + + tasklet_kill(&c->vc.task); + + vchan_free_chan_resources(&c->vc); + + pm_runtime_put_sync(mtkd->ddev.dev); +} + +static enum dma_status mtk_uart_apdma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + if (!txstate) + return ret; + + dma_set_residue(txstate, c->rx_status); + + return ret; +} + +/* + * dmaengine_prep_slave_single will call the function. and sglen is 1. + * 8250 uart using one ring buffer, and deal with one sg. + */ +static struct dma_async_tx_descriptor *mtk_uart_apdma_prep_slave_sg + (struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sglen, enum dma_transfer_direction dir, + unsigned long tx_flags, void *context) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + struct mtk_uart_apdma_desc *d; + + if (!is_slave_direction(dir) || sglen != 1) + return NULL; + + /* Now allocate and setup the descriptor */ + d = kzalloc(sizeof(*d), GFP_ATOMIC); + if (!d) + return NULL; + + d->avail_len = sg_dma_len(sgl); + d->addr = sg_dma_address(sgl); + c->dir = dir; + + return vchan_tx_prep(&c->vc, &d->vd, tx_flags); +} + +static void mtk_uart_apdma_issue_pending(struct dma_chan *chan) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + struct virt_dma_desc *vd; + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + if (vchan_issue_pending(&c->vc)) { + vd = vchan_next_desc(&c->vc); + c->desc = to_mtk_uart_apdma_desc(&vd->tx); + + if (c->dir == DMA_DEV_TO_MEM) + mtk_uart_apdma_start_rx(c); + else if (c->dir == DMA_MEM_TO_DEV) + mtk_uart_apdma_start_tx(c); + } + + spin_unlock_irqrestore(&c->vc.lock, flags); +} + +static int mtk_uart_apdma_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + + memcpy(&c->cfg, config, sizeof(*config)); + + return 0; +} + +static int mtk_uart_apdma_terminate_all(struct dma_chan *chan) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned long flags; + unsigned int status; + LIST_HEAD(head); + int ret; + + mtk_uart_apdma_write(c, VFF_FLUSH, VFF_FLUSH_B); + + ret = readx_poll_timeout(readl, c->base + VFF_FLUSH, + status, status != VFF_FLUSH_B, 10, 100); + if (ret) + dev_err(c->vc.chan.device->dev, "flush: fail, status=0x%x\n", + mtk_uart_apdma_read(c, VFF_DEBUG_STATUS)); + + /* + * Stop need 3 steps. + * 1. set stop to 1 + * 2. wait en to 0 + * 3. set stop as 0 + */ + mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_B); + ret = readx_poll_timeout(readl, c->base + VFF_EN, + status, !status, 10, 100); + if (ret) + dev_err(c->vc.chan.device->dev, "stop: fail, status=0x%x\n", + mtk_uart_apdma_read(c, VFF_DEBUG_STATUS)); + + mtk_uart_apdma_write(c, VFF_STOP, VFF_STOP_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + + if (c->dir == DMA_DEV_TO_MEM) + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); + else if (c->dir == DMA_MEM_TO_DEV) + mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); + + synchronize_irq(c->irq); + + spin_lock_irqsave(&c->vc.lock, flags); + vchan_get_all_descriptors(&c->vc, &head); + vchan_dma_desc_free_list(&c->vc, &head); + spin_unlock_irqrestore(&c->vc.lock, flags); + + return 0; +} + +static int mtk_uart_apdma_device_pause(struct dma_chan *chan) +{ + struct mtk_chan *c = to_mtk_uart_apdma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + + mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); + mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); + + synchronize_irq(c->irq); + + spin_unlock_irqrestore(&c->vc.lock, flags); + + return 0; +} + +static void mtk_uart_apdma_free(struct mtk_uart_apdmadev *mtkd) +{ + while (!list_empty(&mtkd->ddev.channels)) { + struct mtk_chan *c = list_first_entry(&mtkd->ddev.channels, + struct mtk_chan, vc.chan.device_node); + + list_del(&c->vc.chan.device_node); + tasklet_kill(&c->vc.task); + } +} + +static const struct of_device_id mtk_uart_apdma_match[] = { + { .compatible = "mediatek,mt6577-uart-dma", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, mtk_uart_apdma_match); + +static int mtk_uart_apdma_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct mtk_uart_apdmadev *mtkd; + int bit_mask = 32, rc; + struct resource *res; + struct mtk_chan *c; + unsigned int i; + + mtkd = devm_kzalloc(&pdev->dev, sizeof(*mtkd), GFP_KERNEL); + if (!mtkd) + return -ENOMEM; + + mtkd->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(mtkd->clk)) { + dev_err(&pdev->dev, "No clock specified\n"); + rc = PTR_ERR(mtkd->clk); + return rc; + } + + if (of_property_read_bool(np, "mediatek,dma-33bits")) + mtkd->support_33bits = true; + + if (mtkd->support_33bits) + bit_mask = 33; + + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(bit_mask)); + if (rc) + return rc; + + dma_cap_set(DMA_SLAVE, mtkd->ddev.cap_mask); + mtkd->ddev.device_alloc_chan_resources = + mtk_uart_apdma_alloc_chan_resources; + mtkd->ddev.device_free_chan_resources = + mtk_uart_apdma_free_chan_resources; + mtkd->ddev.device_tx_status = mtk_uart_apdma_tx_status; + mtkd->ddev.device_issue_pending = mtk_uart_apdma_issue_pending; + mtkd->ddev.device_prep_slave_sg = mtk_uart_apdma_prep_slave_sg; + mtkd->ddev.device_config = mtk_uart_apdma_slave_config; + mtkd->ddev.device_pause = mtk_uart_apdma_device_pause; + mtkd->ddev.device_terminate_all = mtk_uart_apdma_terminate_all; + mtkd->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE); + mtkd->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE); + mtkd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + mtkd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + mtkd->ddev.dev = &pdev->dev; + INIT_LIST_HEAD(&mtkd->ddev.channels); + + mtkd->dma_requests = MTK_UART_APDMA_NR_VCHANS; + if (of_property_read_u32(np, "dma-requests", &mtkd->dma_requests)) { + dev_info(&pdev->dev, + "Using %u as missing dma-requests property\n", + MTK_UART_APDMA_NR_VCHANS); + } + + for (i = 0; i < mtkd->dma_requests; i++) { + c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL); + if (!c) { + rc = -ENODEV; + goto err_no_dma; + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + if (!res) { + rc = -ENODEV; + goto err_no_dma; + } + + c->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(c->base)) { + rc = PTR_ERR(c->base); + goto err_no_dma; + } + c->vc.desc_free = mtk_uart_apdma_desc_free; + vchan_init(&c->vc, &mtkd->ddev); + + rc = platform_get_irq(pdev, i); + if (rc < 0) { + dev_err(&pdev->dev, "failed to get IRQ[%d]\n", i); + goto err_no_dma; + } + c->irq = rc; + } + + pm_runtime_enable(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + + rc = dma_async_device_register(&mtkd->ddev); + if (rc) + goto rpm_disable; + + platform_set_drvdata(pdev, mtkd); + + /* Device-tree DMA controller registration */ + rc = of_dma_controller_register(np, of_dma_xlate_by_chan_id, mtkd); + if (rc) + goto dma_remove; + + return rc; + +dma_remove: + dma_async_device_unregister(&mtkd->ddev); +rpm_disable: + pm_runtime_disable(&pdev->dev); +err_no_dma: + mtk_uart_apdma_free(mtkd); + return rc; +} + +static int mtk_uart_apdma_remove(struct platform_device *pdev) +{ + struct mtk_uart_apdmadev *mtkd = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + + mtk_uart_apdma_free(mtkd); + + dma_async_device_unregister(&mtkd->ddev); + + pm_runtime_disable(&pdev->dev); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int mtk_uart_apdma_suspend(struct device *dev) +{ + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + if (!pm_runtime_suspended(dev)) + clk_disable_unprepare(mtkd->clk); + + return 0; +} + +static int mtk_uart_apdma_resume(struct device *dev) +{ + int ret; + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + if (!pm_runtime_suspended(dev)) { + ret = clk_prepare_enable(mtkd->clk); + if (ret) + return ret; + } + + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +#ifdef CONFIG_PM +static int mtk_uart_apdma_runtime_suspend(struct device *dev) +{ + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + clk_disable_unprepare(mtkd->clk); + + return 0; +} + +static int mtk_uart_apdma_runtime_resume(struct device *dev) +{ + int ret; + struct mtk_uart_apdmadev *mtkd = dev_get_drvdata(dev); + + ret = clk_prepare_enable(mtkd->clk); + if (ret) + return ret; + + return 0; +} +#endif /* CONFIG_PM */ + +static const struct dev_pm_ops mtk_uart_apdma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_uart_apdma_suspend, mtk_uart_apdma_resume) + SET_RUNTIME_PM_OPS(mtk_uart_apdma_runtime_suspend, + mtk_uart_apdma_runtime_resume, NULL) +}; + +static struct platform_driver mtk_uart_apdma_driver = { + .probe = mtk_uart_apdma_probe, + .remove = mtk_uart_apdma_remove, + .driver = { + .name = KBUILD_MODNAME, + .pm = &mtk_uart_apdma_pm_ops, + .of_match_table = of_match_ptr(mtk_uart_apdma_match), + }, +}; + +module_platform_driver(mtk_uart_apdma_driver); + +MODULE_DESCRIPTION("MediaTek UART APDMA Controller Driver"); +MODULE_AUTHOR("Long Cheng <long.cheng@mediatek.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c index 730a18d0c6d6..fea8608a7810 100644 --- a/drivers/dma/mic_x100_dma.c +++ b/drivers/dma/mic_x100_dma.c @@ -717,10 +717,8 @@ static int mic_dma_driver_probe(struct mbus_device *mbdev) if (mic_dma_dbg) { mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev), mic_dma_dbg); - if (mic_dma_dev->dbg_dir) - debugfs_create_file("mic_dma_reg", 0444, - mic_dma_dev->dbg_dir, mic_dma_dev, - &mic_dma_reg_fops); + debugfs_create_file("mic_dma_reg", 0444, mic_dma_dev->dbg_dir, + mic_dma_dev, &mic_dma_reg_fops); } return 0; } diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index bb3ccbf90a31..e7d1e12bf464 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -582,18 +582,12 @@ static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev, } struct mmp_tdma_filter_param { - struct device_node *of_node; unsigned int chan_id; }; static bool mmp_tdma_filter_fn(struct dma_chan *chan, void *fn_param) { struct mmp_tdma_filter_param *param = fn_param; - struct mmp_tdma_chan *tdmac = to_mmp_tdma_chan(chan); - struct dma_device *pdma_device = tdmac->chan.device; - - if (pdma_device->dev->of_node != param->of_node) - return false; if (chan->chan_id != param->chan_id) return false; @@ -611,13 +605,13 @@ static struct dma_chan *mmp_tdma_xlate(struct of_phandle_args *dma_spec, if (dma_spec->args_count != 1) return NULL; - param.of_node = ofdma->of_node; param.chan_id = dma_spec->args[0]; if (param.chan_id >= TDMA_CHANNEL_NUM) return NULL; - return dma_request_channel(mask, mmp_tdma_filter_fn, ¶m); + return __dma_request_channel(&mask, mmp_tdma_filter_fn, ¶m, + ofdma->of_node); } static const struct of_device_id mmp_tdma_dt_ids[] = { diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index 20a9cb7cb6d3..3039bba0e4d5 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -719,7 +719,6 @@ err_out: } struct mxs_dma_filter_param { - struct device_node *of_node; unsigned int chan_id; }; @@ -730,9 +729,6 @@ static bool mxs_dma_filter_fn(struct dma_chan *chan, void *fn_param) struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; int chan_irq; - if (mxs_dma->dma_device.dev->of_node != param->of_node) - return false; - if (chan->chan_id != param->chan_id) return false; @@ -755,13 +751,13 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec, if (dma_spec->args_count != 1) return NULL; - param.of_node = ofdma->of_node; param.chan_id = dma_spec->args[0]; if (param.chan_id >= mxs_dma->nr_channels) return NULL; - return dma_request_channel(mask, mxs_dma_filter_fn, ¶m); + return __dma_request_channel(&mask, mxs_dma_filter_fn, ¶m, + ofdma->of_node); } static int __init mxs_dma_probe(struct platform_device *pdev) diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c index 1e4d9ef2aea1..c2d779daa4b5 100644 --- a/drivers/dma/of-dma.c +++ b/drivers/dma/of-dma.c @@ -313,8 +313,8 @@ struct dma_chan *of_dma_simple_xlate(struct of_phandle_args *dma_spec, if (count != 1) return NULL; - return dma_request_channel(info->dma_cap, info->filter_fn, - &dma_spec->args[0]); + return __dma_request_channel(&info->dma_cap, info->filter_fn, + &dma_spec->args[0], dma_spec->np); } EXPORT_SYMBOL_GPL(of_dma_simple_xlate); diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 56f9fabc99c4..1163af2ba4a3 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -25,6 +25,7 @@ #include <linux/err.h> #include <linux/pm_runtime.h> #include <linux/bug.h> +#include <linux/reset.h> #include "dmaengine.h" #define PL330_MAX_CHAN 8 @@ -496,6 +497,9 @@ struct pl330_dmac { unsigned int num_peripherals; struct dma_pl330_chan *peripherals; /* keep at end */ int quirks; + + struct reset_control *rstc; + struct reset_control *rstc_ocp; }; static struct pl330_of_quirks { @@ -3024,6 +3028,32 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) amba_set_drvdata(adev, pl330); + pl330->rstc = devm_reset_control_get_optional(&adev->dev, "dma"); + if (IS_ERR(pl330->rstc)) { + if (PTR_ERR(pl330->rstc) != -EPROBE_DEFER) + dev_err(&adev->dev, "Failed to get reset!\n"); + return PTR_ERR(pl330->rstc); + } else { + ret = reset_control_deassert(pl330->rstc); + if (ret) { + dev_err(&adev->dev, "Couldn't deassert the device from reset!\n"); + return ret; + } + } + + pl330->rstc_ocp = devm_reset_control_get_optional(&adev->dev, "dma-ocp"); + if (IS_ERR(pl330->rstc_ocp)) { + if (PTR_ERR(pl330->rstc_ocp) != -EPROBE_DEFER) + dev_err(&adev->dev, "Failed to get OCP reset!\n"); + return PTR_ERR(pl330->rstc_ocp); + } else { + ret = reset_control_deassert(pl330->rstc_ocp); + if (ret) { + dev_err(&adev->dev, "Couldn't deassert the device from OCP reset!\n"); + return ret; + } + } + for (i = 0; i < AMBA_NR_IRQS; i++) { irq = adev->irq[i]; if (irq) { @@ -3164,6 +3194,11 @@ probe_err3: probe_err2: pl330_del(pl330); + if (pl330->rstc_ocp) + reset_control_assert(pl330->rstc_ocp); + + if (pl330->rstc) + reset_control_assert(pl330->rstc); return ret; } @@ -3202,6 +3237,11 @@ static int pl330_remove(struct amba_device *adev) pl330_del(pl330); + if (pl330->rstc_ocp) + reset_control_assert(pl330->rstc_ocp); + + if (pl330->rstc) + reset_control_assert(pl330->rstc); return 0; } diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index 468c234cb3be..349fb312c872 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -129,7 +129,6 @@ struct pxad_device { spinlock_t phy_lock; /* Phy association */ #ifdef CONFIG_DEBUG_FS struct dentry *dbgfs_root; - struct dentry *dbgfs_state; struct dentry **dbgfs_chan; #endif }; @@ -323,31 +322,18 @@ static struct dentry *pxad_dbg_alloc_chan(struct pxad_device *pdev, int ch, struct dentry *chandir) { char chan_name[11]; - struct dentry *chan, *chan_state = NULL, *chan_descr = NULL; - struct dentry *chan_reqs = NULL; + struct dentry *chan; void *dt; scnprintf(chan_name, sizeof(chan_name), "%d", ch); chan = debugfs_create_dir(chan_name, chandir); dt = (void *)&pdev->phys[ch]; - if (chan) - chan_state = debugfs_create_file("state", 0400, chan, dt, - &chan_state_fops); - if (chan_state) - chan_descr = debugfs_create_file("descriptors", 0400, chan, dt, - &descriptors_fops); - if (chan_descr) - chan_reqs = debugfs_create_file("requesters", 0400, chan, dt, - &requester_chan_fops); - if (!chan_reqs) - goto err_state; + debugfs_create_file("state", 0400, chan, dt, &chan_state_fops); + debugfs_create_file("descriptors", 0400, chan, dt, &descriptors_fops); + debugfs_create_file("requesters", 0400, chan, dt, &requester_chan_fops); return chan; - -err_state: - debugfs_remove_recursive(chan); - return NULL; } static void pxad_init_debugfs(struct pxad_device *pdev) @@ -355,40 +341,20 @@ static void pxad_init_debugfs(struct pxad_device *pdev) int i; struct dentry *chandir; - pdev->dbgfs_root = debugfs_create_dir(dev_name(pdev->slave.dev), NULL); - if (IS_ERR(pdev->dbgfs_root) || !pdev->dbgfs_root) - goto err_root; - - pdev->dbgfs_state = debugfs_create_file("state", 0400, pdev->dbgfs_root, - pdev, &state_fops); - if (!pdev->dbgfs_state) - goto err_state; - pdev->dbgfs_chan = - kmalloc_array(pdev->nr_chans, sizeof(*pdev->dbgfs_state), + kmalloc_array(pdev->nr_chans, sizeof(struct dentry *), GFP_KERNEL); if (!pdev->dbgfs_chan) - goto err_alloc; + return; + + pdev->dbgfs_root = debugfs_create_dir(dev_name(pdev->slave.dev), NULL); + + debugfs_create_file("state", 0400, pdev->dbgfs_root, pdev, &state_fops); chandir = debugfs_create_dir("channels", pdev->dbgfs_root); - if (!chandir) - goto err_chandir; - for (i = 0; i < pdev->nr_chans; i++) { + for (i = 0; i < pdev->nr_chans; i++) pdev->dbgfs_chan[i] = pxad_dbg_alloc_chan(pdev, i, chandir); - if (!pdev->dbgfs_chan[i]) - goto err_chans; - } - - return; -err_chans: -err_chandir: - kfree(pdev->dbgfs_chan); -err_alloc: -err_state: - debugfs_remove_recursive(pdev->dbgfs_root); -err_root: - pr_err("pxad: debugfs is not available\n"); } static void pxad_cleanup_debugfs(struct pxad_device *pdev) diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h index f337e2789ddc..f212466744f3 100644 --- a/drivers/dma/qcom/hidma.h +++ b/drivers/dma/qcom/hidma.h @@ -93,8 +93,6 @@ struct hidma_chan { * It is used by the DMA complete notification to * locate the descriptor that initiated the transfer. */ - struct dentry *debugfs; - struct dentry *stats; struct hidma_dev *dmadev; struct hidma_desc *running; @@ -126,7 +124,6 @@ struct hidma_dev { struct dma_device ddev; struct dentry *debugfs; - struct dentry *stats; /* sysfs entry for the channel id */ struct device_attribute *chid_attrs; @@ -158,6 +155,6 @@ irqreturn_t hidma_ll_inthandler(int irq, void *arg); irqreturn_t hidma_ll_inthandler_msi(int irq, void *arg, int cause); void hidma_cleanup_pending_tre(struct hidma_lldev *llhndl, u8 err_info, u8 err_code); -int hidma_debug_init(struct hidma_dev *dmadev); +void hidma_debug_init(struct hidma_dev *dmadev); void hidma_debug_uninit(struct hidma_dev *dmadev); #endif diff --git a/drivers/dma/qcom/hidma_dbg.c b/drivers/dma/qcom/hidma_dbg.c index 75b0691a670d..ce87c7937a0e 100644 --- a/drivers/dma/qcom/hidma_dbg.c +++ b/drivers/dma/qcom/hidma_dbg.c @@ -138,17 +138,13 @@ void hidma_debug_uninit(struct hidma_dev *dmadev) debugfs_remove_recursive(dmadev->debugfs); } -int hidma_debug_init(struct hidma_dev *dmadev) +void hidma_debug_init(struct hidma_dev *dmadev) { - int rc = 0; int chidx = 0; struct list_head *position = NULL; + struct dentry *dir; dmadev->debugfs = debugfs_create_dir(dev_name(dmadev->ddev.dev), NULL); - if (!dmadev->debugfs) { - rc = -ENODEV; - return rc; - } /* walk through the virtual channel list */ list_for_each(position, &dmadev->ddev.channels) { @@ -157,32 +153,13 @@ int hidma_debug_init(struct hidma_dev *dmadev) chan = list_entry(position, struct hidma_chan, chan.device_node); sprintf(chan->dbg_name, "chan%d", chidx); - chan->debugfs = debugfs_create_dir(chan->dbg_name, + dir = debugfs_create_dir(chan->dbg_name, dmadev->debugfs); - if (!chan->debugfs) { - rc = -ENOMEM; - goto cleanup; - } - chan->stats = debugfs_create_file("stats", S_IRUGO, - chan->debugfs, chan, - &hidma_chan_fops); - if (!chan->stats) { - rc = -ENOMEM; - goto cleanup; - } + debugfs_create_file("stats", S_IRUGO, dir, chan, + &hidma_chan_fops); chidx++; } - dmadev->stats = debugfs_create_file("stats", S_IRUGO, - dmadev->debugfs, dmadev, - &hidma_dma_fops); - if (!dmadev->stats) { - rc = -ENOMEM; - goto cleanup; - } - - return 0; -cleanup: - hidma_debug_uninit(dmadev); - return rc; + debugfs_create_file("stats", S_IRUGO, dmadev->debugfs, dmadev, + &hidma_dma_fops); } diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig index 4d6b02b3b1f1..54d5d0369d3c 100644 --- a/drivers/dma/sh/Kconfig +++ b/drivers/dma/sh/Kconfig @@ -47,9 +47,3 @@ config RENESAS_USB_DMAC help This driver supports the USB-DMA controller found in the Renesas SoCs. - -config SUDMAC - tristate "Renesas SUDMAC support" - depends on SH_DMAE_BASE - help - Enable support for the Renesas SUDMAC controllers. diff --git a/drivers/dma/sh/Makefile b/drivers/dma/sh/Makefile index 42110dd57a56..112fbd22bb3f 100644 --- a/drivers/dma/sh/Makefile +++ b/drivers/dma/sh/Makefile @@ -15,4 +15,3 @@ obj-$(CONFIG_SH_DMAE) += shdma.o obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o -obj-$(CONFIG_SUDMAC) += sudmac.o diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 33ab1b607e2b..9c41a4e42575 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -1165,7 +1165,7 @@ rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); /* Someone calling slave DMA on a generic channel? */ - if (rchan->mid_rid < 0 || !sg_len) { + if (rchan->mid_rid < 0 || !sg_len || !sg_dma_len(sgl)) { dev_warn(chan->device->dev, "%s: bad parameter: len=%d, id=%d\n", __func__, sg_len, rchan->mid_rid); @@ -1654,8 +1654,7 @@ static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg) * Forcing it to call dma_request_channel() and iterate through all * channels from all controllers is just pointless. */ - if (chan->device->device_config != rcar_dmac_device_config || - dma_spec->np != chan->device->dev->of_node) + if (chan->device->device_config != rcar_dmac_device_config) return false; return !test_and_set_bit(dma_spec->args[0], dmac->modules); @@ -1675,7 +1674,8 @@ static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); - chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec); + chan = __dma_request_channel(&mask, rcar_dmac_chan_filter, dma_spec, + ofdma->of_node); if (!chan) return NULL; diff --git a/drivers/dma/sh/sudmac.c b/drivers/dma/sh/sudmac.c deleted file mode 100644 index 30cc3553cb8b..000000000000 --- a/drivers/dma/sh/sudmac.c +++ /dev/null @@ -1,414 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Renesas SUDMAC support - * - * Copyright (C) 2013 Renesas Solutions Corp. - * - * based on drivers/dma/sh/shdma.c: - * Copyright (C) 2011-2012 Guennadi Liakhovetski <g.liakhovetski@gmx.de> - * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com> - * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved. - * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. - */ - -#include <linux/dmaengine.h> -#include <linux/err.h> -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/module.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/sudmac.h> - -struct sudmac_chan { - struct shdma_chan shdma_chan; - void __iomem *base; - char dev_id[16]; /* unique name per DMAC of channel */ - - u32 offset; /* for CFG, BA, BBC, CA, CBC, DEN */ - u32 cfg; - u32 dint_end_bit; -}; - -struct sudmac_device { - struct shdma_dev shdma_dev; - struct sudmac_pdata *pdata; - void __iomem *chan_reg; -}; - -struct sudmac_regs { - u32 base_addr; - u32 base_byte_count; -}; - -struct sudmac_desc { - struct sudmac_regs hw; - struct shdma_desc shdma_desc; -}; - -#define to_chan(schan) container_of(schan, struct sudmac_chan, shdma_chan) -#define to_desc(sdesc) container_of(sdesc, struct sudmac_desc, shdma_desc) -#define to_sdev(sc) container_of(sc->shdma_chan.dma_chan.device, \ - struct sudmac_device, shdma_dev.dma_dev) - -/* SUDMAC register */ -#define SUDMAC_CH0CFG 0x00 -#define SUDMAC_CH0BA 0x10 -#define SUDMAC_CH0BBC 0x18 -#define SUDMAC_CH0CA 0x20 -#define SUDMAC_CH0CBC 0x28 -#define SUDMAC_CH0DEN 0x30 -#define SUDMAC_DSTSCLR 0x38 -#define SUDMAC_DBUFCTRL 0x3C -#define SUDMAC_DINTCTRL 0x40 -#define SUDMAC_DINTSTS 0x44 -#define SUDMAC_DINTSTSCLR 0x48 -#define SUDMAC_CH0SHCTRL 0x50 - -/* Definitions for the sudmac_channel.config */ -#define SUDMAC_SENDBUFM 0x1000 /* b12: Transmit Buffer Mode */ -#define SUDMAC_RCVENDM 0x0100 /* b8: Receive Data Transfer End Mode */ -#define SUDMAC_LBA_WAIT 0x0030 /* b5-4: Local Bus Access Wait */ - -/* Definitions for the sudmac_channel.dint_end_bit */ -#define SUDMAC_CH1ENDE 0x0002 /* b1: Ch1 DMA Transfer End Int Enable */ -#define SUDMAC_CH0ENDE 0x0001 /* b0: Ch0 DMA Transfer End Int Enable */ - -#define SUDMAC_DRV_NAME "sudmac" - -static void sudmac_writel(struct sudmac_chan *sc, u32 data, u32 reg) -{ - iowrite32(data, sc->base + reg); -} - -static u32 sudmac_readl(struct sudmac_chan *sc, u32 reg) -{ - return ioread32(sc->base + reg); -} - -static bool sudmac_is_busy(struct sudmac_chan *sc) -{ - u32 den = sudmac_readl(sc, SUDMAC_CH0DEN + sc->offset); - - if (den) - return true; /* working */ - - return false; /* waiting */ -} - -static void sudmac_set_reg(struct sudmac_chan *sc, struct sudmac_regs *hw, - struct shdma_desc *sdesc) -{ - sudmac_writel(sc, sc->cfg, SUDMAC_CH0CFG + sc->offset); - sudmac_writel(sc, hw->base_addr, SUDMAC_CH0BA + sc->offset); - sudmac_writel(sc, hw->base_byte_count, SUDMAC_CH0BBC + sc->offset); -} - -static void sudmac_start(struct sudmac_chan *sc) -{ - u32 dintctrl = sudmac_readl(sc, SUDMAC_DINTCTRL); - - sudmac_writel(sc, dintctrl | sc->dint_end_bit, SUDMAC_DINTCTRL); - sudmac_writel(sc, 1, SUDMAC_CH0DEN + sc->offset); -} - -static void sudmac_start_xfer(struct shdma_chan *schan, - struct shdma_desc *sdesc) -{ - struct sudmac_chan *sc = to_chan(schan); - struct sudmac_desc *sd = to_desc(sdesc); - - sudmac_set_reg(sc, &sd->hw, sdesc); - sudmac_start(sc); -} - -static bool sudmac_channel_busy(struct shdma_chan *schan) -{ - struct sudmac_chan *sc = to_chan(schan); - - return sudmac_is_busy(sc); -} - -static void sudmac_setup_xfer(struct shdma_chan *schan, int slave_id) -{ -} - -static const struct sudmac_slave_config *sudmac_find_slave( - struct sudmac_chan *sc, int slave_id) -{ - struct sudmac_device *sdev = to_sdev(sc); - struct sudmac_pdata *pdata = sdev->pdata; - const struct sudmac_slave_config *cfg; - int i; - - for (i = 0, cfg = pdata->slave; i < pdata->slave_num; i++, cfg++) - if (cfg->slave_id == slave_id) - return cfg; - - return NULL; -} - -static int sudmac_set_slave(struct shdma_chan *schan, int slave_id, - dma_addr_t slave_addr, bool try) -{ - struct sudmac_chan *sc = to_chan(schan); - const struct sudmac_slave_config *cfg = sudmac_find_slave(sc, slave_id); - - if (!cfg) - return -ENODEV; - - return 0; -} - -static inline void sudmac_dma_halt(struct sudmac_chan *sc) -{ - u32 dintctrl = sudmac_readl(sc, SUDMAC_DINTCTRL); - - sudmac_writel(sc, 0, SUDMAC_CH0DEN + sc->offset); - sudmac_writel(sc, dintctrl & ~sc->dint_end_bit, SUDMAC_DINTCTRL); - sudmac_writel(sc, sc->dint_end_bit, SUDMAC_DINTSTSCLR); -} - -static int sudmac_desc_setup(struct shdma_chan *schan, - struct shdma_desc *sdesc, - dma_addr_t src, dma_addr_t dst, size_t *len) -{ - struct sudmac_chan *sc = to_chan(schan); - struct sudmac_desc *sd = to_desc(sdesc); - - dev_dbg(sc->shdma_chan.dev, "%s: src=%pad, dst=%pad, len=%zu\n", - __func__, &src, &dst, *len); - - if (*len > schan->max_xfer_len) - *len = schan->max_xfer_len; - - if (dst) - sd->hw.base_addr = dst; - else if (src) - sd->hw.base_addr = src; - sd->hw.base_byte_count = *len; - - return 0; -} - -static void sudmac_halt(struct shdma_chan *schan) -{ - struct sudmac_chan *sc = to_chan(schan); - - sudmac_dma_halt(sc); -} - -static bool sudmac_chan_irq(struct shdma_chan *schan, int irq) -{ - struct sudmac_chan *sc = to_chan(schan); - u32 dintsts = sudmac_readl(sc, SUDMAC_DINTSTS); - - if (!(dintsts & sc->dint_end_bit)) - return false; - - /* DMA stop */ - sudmac_dma_halt(sc); - - return true; -} - -static size_t sudmac_get_partial(struct shdma_chan *schan, - struct shdma_desc *sdesc) -{ - struct sudmac_chan *sc = to_chan(schan); - struct sudmac_desc *sd = to_desc(sdesc); - u32 current_byte_count = sudmac_readl(sc, SUDMAC_CH0CBC + sc->offset); - - return sd->hw.base_byte_count - current_byte_count; -} - -static bool sudmac_desc_completed(struct shdma_chan *schan, - struct shdma_desc *sdesc) -{ - struct sudmac_chan *sc = to_chan(schan); - struct sudmac_desc *sd = to_desc(sdesc); - u32 current_addr = sudmac_readl(sc, SUDMAC_CH0CA + sc->offset); - - return sd->hw.base_addr + sd->hw.base_byte_count == current_addr; -} - -static int sudmac_chan_probe(struct sudmac_device *su_dev, int id, int irq, - unsigned long flags) -{ - struct shdma_dev *sdev = &su_dev->shdma_dev; - struct platform_device *pdev = to_platform_device(sdev->dma_dev.dev); - struct sudmac_chan *sc; - struct shdma_chan *schan; - int err; - - sc = devm_kzalloc(&pdev->dev, sizeof(struct sudmac_chan), GFP_KERNEL); - if (!sc) - return -ENOMEM; - - schan = &sc->shdma_chan; - schan->max_xfer_len = 64 * 1024 * 1024 - 1; - - shdma_chan_probe(sdev, schan, id); - - sc->base = su_dev->chan_reg; - - /* get platform_data */ - sc->offset = su_dev->pdata->channel->offset; - if (su_dev->pdata->channel->config & SUDMAC_TX_BUFFER_MODE) - sc->cfg |= SUDMAC_SENDBUFM; - if (su_dev->pdata->channel->config & SUDMAC_RX_END_MODE) - sc->cfg |= SUDMAC_RCVENDM; - sc->cfg |= (su_dev->pdata->channel->wait << 4) & SUDMAC_LBA_WAIT; - - if (su_dev->pdata->channel->dint_end_bit & SUDMAC_DMA_BIT_CH0) - sc->dint_end_bit |= SUDMAC_CH0ENDE; - if (su_dev->pdata->channel->dint_end_bit & SUDMAC_DMA_BIT_CH1) - sc->dint_end_bit |= SUDMAC_CH1ENDE; - - /* set up channel irq */ - if (pdev->id >= 0) - snprintf(sc->dev_id, sizeof(sc->dev_id), "sudmac%d.%d", - pdev->id, id); - else - snprintf(sc->dev_id, sizeof(sc->dev_id), "sudmac%d", id); - - err = shdma_request_irq(schan, irq, flags, sc->dev_id); - if (err) { - dev_err(sdev->dma_dev.dev, - "DMA channel %d request_irq failed %d\n", id, err); - goto err_no_irq; - } - - return 0; - -err_no_irq: - /* remove from dmaengine device node */ - shdma_chan_remove(schan); - return err; -} - -static void sudmac_chan_remove(struct sudmac_device *su_dev) -{ - struct shdma_chan *schan; - int i; - - shdma_for_each_chan(schan, &su_dev->shdma_dev, i) { - BUG_ON(!schan); - - shdma_chan_remove(schan); - } -} - -static dma_addr_t sudmac_slave_addr(struct shdma_chan *schan) -{ - /* SUDMAC doesn't need the address */ - return 0; -} - -static struct shdma_desc *sudmac_embedded_desc(void *buf, int i) -{ - return &((struct sudmac_desc *)buf)[i].shdma_desc; -} - -static const struct shdma_ops sudmac_shdma_ops = { - .desc_completed = sudmac_desc_completed, - .halt_channel = sudmac_halt, - .channel_busy = sudmac_channel_busy, - .slave_addr = sudmac_slave_addr, - .desc_setup = sudmac_desc_setup, - .set_slave = sudmac_set_slave, - .setup_xfer = sudmac_setup_xfer, - .start_xfer = sudmac_start_xfer, - .embedded_desc = sudmac_embedded_desc, - .chan_irq = sudmac_chan_irq, - .get_partial = sudmac_get_partial, -}; - -static int sudmac_probe(struct platform_device *pdev) -{ - struct sudmac_pdata *pdata = dev_get_platdata(&pdev->dev); - int err, i; - struct sudmac_device *su_dev; - struct dma_device *dma_dev; - struct resource *chan, *irq_res; - - /* get platform data */ - if (!pdata) - return -ENODEV; - - irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!irq_res) - return -ENODEV; - - err = -ENOMEM; - su_dev = devm_kzalloc(&pdev->dev, sizeof(struct sudmac_device), - GFP_KERNEL); - if (!su_dev) - return err; - - dma_dev = &su_dev->shdma_dev.dma_dev; - - chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); - su_dev->chan_reg = devm_ioremap_resource(&pdev->dev, chan); - if (IS_ERR(su_dev->chan_reg)) - return PTR_ERR(su_dev->chan_reg); - - dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); - - su_dev->shdma_dev.ops = &sudmac_shdma_ops; - su_dev->shdma_dev.desc_size = sizeof(struct sudmac_desc); - err = shdma_init(&pdev->dev, &su_dev->shdma_dev, pdata->channel_num); - if (err < 0) - return err; - - /* platform data */ - su_dev->pdata = dev_get_platdata(&pdev->dev); - - platform_set_drvdata(pdev, su_dev); - - /* Create DMA Channel */ - for (i = 0; i < pdata->channel_num; i++) { - err = sudmac_chan_probe(su_dev, i, irq_res->start, IRQF_SHARED); - if (err) - goto chan_probe_err; - } - - err = dma_async_device_register(&su_dev->shdma_dev.dma_dev); - if (err < 0) - goto chan_probe_err; - - return err; - -chan_probe_err: - sudmac_chan_remove(su_dev); - - shdma_cleanup(&su_dev->shdma_dev); - - return err; -} - -static int sudmac_remove(struct platform_device *pdev) -{ - struct sudmac_device *su_dev = platform_get_drvdata(pdev); - struct dma_device *dma_dev = &su_dev->shdma_dev.dma_dev; - - dma_async_device_unregister(dma_dev); - sudmac_chan_remove(su_dev); - shdma_cleanup(&su_dev->shdma_dev); - - return 0; -} - -static struct platform_driver sudmac_driver = { - .driver = { - .name = SUDMAC_DRV_NAME, - }, - .probe = sudmac_probe, - .remove = sudmac_remove, -}; -module_platform_driver(sudmac_driver); - -MODULE_AUTHOR("Yoshihiro Shimoda"); -MODULE_DESCRIPTION("Renesas SUDMAC driver"); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:" SUDMAC_DRV_NAME); diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c index 59403f6d008a..17063aaf51bc 100644 --- a/drivers/dma/sh/usb-dmac.c +++ b/drivers/dma/sh/usb-dmac.c @@ -57,7 +57,7 @@ struct usb_dmac_desc { u32 residue; struct list_head node; dma_cookie_t done_cookie; - struct usb_dmac_sg sg[0]; + struct usb_dmac_sg sg[]; }; #define to_usb_dmac_desc(vd) container_of(vd, struct usb_dmac_desc, vd) @@ -636,9 +636,6 @@ static bool usb_dmac_chan_filter(struct dma_chan *chan, void *arg) struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); struct of_phandle_args *dma_spec = arg; - if (dma_spec->np != chan->device->dev->of_node) - return false; - /* USB-DMAC should be used with fixed usb controller's FIFO */ if (uchan->index != dma_spec->args[0]) return false; @@ -659,7 +656,8 @@ static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec, dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); - chan = dma_request_channel(mask, usb_dmac_chan_filter, dma_spec); + chan = __dma_request_channel(&mask, usb_dmac_chan_filter, dma_spec, + ofdma->of_node); if (!chan) return NULL; diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index da41bab98f5b..ef4d109e7189 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -1365,7 +1365,6 @@ static int stm32_dma_probe(struct platform_device *pdev) for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) { chan = &dmadev->chan[i]; - chan->irq = platform_get_irq(pdev, i); ret = platform_get_irq(pdev, i); if (ret < 0) { if (ret != -EPROBE_DEFER) diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c index 715aad7a9192..b552949da14b 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32-dmamux.c @@ -295,8 +295,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev) #ifdef CONFIG_PM static int stm32_dmamux_runtime_suspend(struct device *dev) { - struct platform_device *pdev = - container_of(dev, struct platform_device, dev); + struct platform_device *pdev = to_platform_device(dev); struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); clk_disable_unprepare(stm32_dmamux->clk); @@ -306,8 +305,7 @@ static int stm32_dmamux_runtime_suspend(struct device *dev) static int stm32_dmamux_runtime_resume(struct device *dev) { - struct platform_device *pdev = - container_of(dev, struct platform_device, dev); + struct platform_device *pdev = to_platform_device(dev); struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); int ret; diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index e8fcc69b1de9..ed5b68dcfe50 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -64,17 +64,20 @@ #define DMA_CHAN_LLI_ADDR 0x08 #define DMA_CHAN_CUR_CFG 0x0c -#define DMA_CHAN_MAX_DRQ 0x1f -#define DMA_CHAN_CFG_SRC_DRQ(x) ((x) & DMA_CHAN_MAX_DRQ) -#define DMA_CHAN_CFG_SRC_IO_MODE BIT(5) -#define DMA_CHAN_CFG_SRC_LINEAR_MODE (0 << 5) +#define DMA_CHAN_MAX_DRQ_A31 0x1f +#define DMA_CHAN_MAX_DRQ_H6 0x3f +#define DMA_CHAN_CFG_SRC_DRQ_A31(x) ((x) & DMA_CHAN_MAX_DRQ_A31) +#define DMA_CHAN_CFG_SRC_DRQ_H6(x) ((x) & DMA_CHAN_MAX_DRQ_H6) +#define DMA_CHAN_CFG_SRC_MODE_A31(x) (((x) & 0x1) << 5) +#define DMA_CHAN_CFG_SRC_MODE_H6(x) (((x) & 0x1) << 8) #define DMA_CHAN_CFG_SRC_BURST_A31(x) (((x) & 0x3) << 7) #define DMA_CHAN_CFG_SRC_BURST_H3(x) (((x) & 0x3) << 6) #define DMA_CHAN_CFG_SRC_WIDTH(x) (((x) & 0x3) << 9) -#define DMA_CHAN_CFG_DST_DRQ(x) (DMA_CHAN_CFG_SRC_DRQ(x) << 16) -#define DMA_CHAN_CFG_DST_IO_MODE (DMA_CHAN_CFG_SRC_IO_MODE << 16) -#define DMA_CHAN_CFG_DST_LINEAR_MODE (DMA_CHAN_CFG_SRC_LINEAR_MODE << 16) +#define DMA_CHAN_CFG_DST_DRQ_A31(x) (DMA_CHAN_CFG_SRC_DRQ_A31(x) << 16) +#define DMA_CHAN_CFG_DST_DRQ_H6(x) (DMA_CHAN_CFG_SRC_DRQ_H6(x) << 16) +#define DMA_CHAN_CFG_DST_MODE_A31(x) (DMA_CHAN_CFG_SRC_MODE_A31(x) << 16) +#define DMA_CHAN_CFG_DST_MODE_H6(x) (DMA_CHAN_CFG_SRC_MODE_H6(x) << 16) #define DMA_CHAN_CFG_DST_BURST_A31(x) (DMA_CHAN_CFG_SRC_BURST_A31(x) << 16) #define DMA_CHAN_CFG_DST_BURST_H3(x) (DMA_CHAN_CFG_SRC_BURST_H3(x) << 16) #define DMA_CHAN_CFG_DST_WIDTH(x) (DMA_CHAN_CFG_SRC_WIDTH(x) << 16) @@ -94,6 +97,8 @@ #define LLI_LAST_ITEM 0xfffff800 #define NORMAL_WAIT 8 #define DRQ_SDRAM 1 +#define LINEAR_MODE 0 +#define IO_MODE 1 /* forward declaration */ struct sun6i_dma_dev; @@ -121,10 +126,13 @@ struct sun6i_dma_config { */ void (*clock_autogate_enable)(struct sun6i_dma_dev *); void (*set_burst_length)(u32 *p_cfg, s8 src_burst, s8 dst_burst); + void (*set_drq)(u32 *p_cfg, s8 src_drq, s8 dst_drq); + void (*set_mode)(u32 *p_cfg, s8 src_mode, s8 dst_mode); u32 src_burst_lengths; u32 dst_burst_lengths; u32 src_addr_widths; u32 dst_addr_widths; + bool has_mbus_clk; }; /* @@ -178,6 +186,7 @@ struct sun6i_dma_dev { struct dma_device slave; void __iomem *base; struct clk *clk; + struct clk *clk_mbus; int irq; spinlock_t lock; struct reset_control *rstc; @@ -305,6 +314,30 @@ static void sun6i_set_burst_length_h3(u32 *p_cfg, s8 src_burst, s8 dst_burst) DMA_CHAN_CFG_DST_BURST_H3(dst_burst); } +static void sun6i_set_drq_a31(u32 *p_cfg, s8 src_drq, s8 dst_drq) +{ + *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_A31(src_drq) | + DMA_CHAN_CFG_DST_DRQ_A31(dst_drq); +} + +static void sun6i_set_drq_h6(u32 *p_cfg, s8 src_drq, s8 dst_drq) +{ + *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_H6(src_drq) | + DMA_CHAN_CFG_DST_DRQ_H6(dst_drq); +} + +static void sun6i_set_mode_a31(u32 *p_cfg, s8 src_mode, s8 dst_mode) +{ + *p_cfg |= DMA_CHAN_CFG_SRC_MODE_A31(src_mode) | + DMA_CHAN_CFG_DST_MODE_A31(dst_mode); +} + +static void sun6i_set_mode_h6(u32 *p_cfg, s8 src_mode, s8 dst_mode) +{ + *p_cfg |= DMA_CHAN_CFG_SRC_MODE_H6(src_mode) | + DMA_CHAN_CFG_DST_MODE_H6(dst_mode); +} + static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan) { struct sun6i_desc *txd = pchan->desc; @@ -628,14 +661,12 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( burst = convert_burst(8); width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES); - v_lli->cfg = DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) | - DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) | - DMA_CHAN_CFG_DST_LINEAR_MODE | - DMA_CHAN_CFG_SRC_LINEAR_MODE | - DMA_CHAN_CFG_SRC_WIDTH(width) | + v_lli->cfg = DMA_CHAN_CFG_SRC_WIDTH(width) | DMA_CHAN_CFG_DST_WIDTH(width); sdev->cfg->set_burst_length(&v_lli->cfg, burst, burst); + sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, DRQ_SDRAM); + sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, LINEAR_MODE); sun6i_dma_lli_add(NULL, v_lli, p_lli, txd); @@ -687,11 +718,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( if (dir == DMA_MEM_TO_DEV) { v_lli->src = sg_dma_address(sg); v_lli->dst = sconfig->dst_addr; - v_lli->cfg = lli_cfg | - DMA_CHAN_CFG_DST_IO_MODE | - DMA_CHAN_CFG_SRC_LINEAR_MODE | - DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) | - DMA_CHAN_CFG_DST_DRQ(vchan->port); + v_lli->cfg = lli_cfg; + sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); + sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); dev_dbg(chan2dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", @@ -702,11 +731,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( } else { v_lli->src = sconfig->src_addr; v_lli->dst = sg_dma_address(sg); - v_lli->cfg = lli_cfg | - DMA_CHAN_CFG_DST_LINEAR_MODE | - DMA_CHAN_CFG_SRC_IO_MODE | - DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) | - DMA_CHAN_CFG_SRC_DRQ(vchan->port); + v_lli->cfg = lli_cfg; + sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); + sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); dev_dbg(chan2dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", @@ -772,19 +799,15 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( if (dir == DMA_MEM_TO_DEV) { v_lli->src = buf_addr + period_len * i; v_lli->dst = sconfig->dst_addr; - v_lli->cfg = lli_cfg | - DMA_CHAN_CFG_DST_IO_MODE | - DMA_CHAN_CFG_SRC_LINEAR_MODE | - DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) | - DMA_CHAN_CFG_DST_DRQ(vchan->port); + v_lli->cfg = lli_cfg; + sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); + sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); } else { v_lli->src = sconfig->src_addr; v_lli->dst = buf_addr + period_len * i; - v_lli->cfg = lli_cfg | - DMA_CHAN_CFG_DST_LINEAR_MODE | - DMA_CHAN_CFG_SRC_IO_MODE | - DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) | - DMA_CHAN_CFG_SRC_DRQ(vchan->port); + v_lli->cfg = lli_cfg; + sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); + sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); } prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); @@ -1049,6 +1072,8 @@ static struct sun6i_dma_config sun6i_a31_dma_cfg = { .nr_max_requests = 30, .nr_max_vchans = 53, .set_burst_length = sun6i_set_burst_length_a31, + .set_drq = sun6i_set_drq_a31, + .set_mode = sun6i_set_mode_a31, .src_burst_lengths = BIT(1) | BIT(8), .dst_burst_lengths = BIT(1) | BIT(8), .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | @@ -1070,6 +1095,8 @@ static struct sun6i_dma_config sun8i_a23_dma_cfg = { .nr_max_vchans = 37, .clock_autogate_enable = sun6i_enable_clock_autogate_a23, .set_burst_length = sun6i_set_burst_length_a31, + .set_drq = sun6i_set_drq_a31, + .set_mode = sun6i_set_mode_a31, .src_burst_lengths = BIT(1) | BIT(8), .dst_burst_lengths = BIT(1) | BIT(8), .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | @@ -1086,6 +1113,8 @@ static struct sun6i_dma_config sun8i_a83t_dma_cfg = { .nr_max_vchans = 39, .clock_autogate_enable = sun6i_enable_clock_autogate_a23, .set_burst_length = sun6i_set_burst_length_a31, + .set_drq = sun6i_set_drq_a31, + .set_mode = sun6i_set_mode_a31, .src_burst_lengths = BIT(1) | BIT(8), .dst_burst_lengths = BIT(1) | BIT(8), .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | @@ -1109,6 +1138,8 @@ static struct sun6i_dma_config sun8i_h3_dma_cfg = { .nr_max_vchans = 34, .clock_autogate_enable = sun6i_enable_clock_autogate_h3, .set_burst_length = sun6i_set_burst_length_h3, + .set_drq = sun6i_set_drq_a31, + .set_mode = sun6i_set_mode_a31, .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | @@ -1128,6 +1159,8 @@ static struct sun6i_dma_config sun8i_h3_dma_cfg = { static struct sun6i_dma_config sun50i_a64_dma_cfg = { .clock_autogate_enable = sun6i_enable_clock_autogate_h3, .set_burst_length = sun6i_set_burst_length_h3, + .set_drq = sun6i_set_drq_a31, + .set_mode = sun6i_set_mode_a31, .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | @@ -1141,6 +1174,28 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = { }; /* + * The H6 binding uses the number of dma channels from the + * device tree node. + */ +static struct sun6i_dma_config sun50i_h6_dma_cfg = { + .clock_autogate_enable = sun6i_enable_clock_autogate_h3, + .set_burst_length = sun6i_set_burst_length_h3, + .set_drq = sun6i_set_drq_h6, + .set_mode = sun6i_set_mode_h6, + .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), + .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), + .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), + .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), + .has_mbus_clk = true, +}; + +/* * The V3s have only 8 physical channels, a maximum DRQ port id of 23, * and a total of 24 usable source and destination endpoints. */ @@ -1151,6 +1206,8 @@ static struct sun6i_dma_config sun8i_v3s_dma_cfg = { .nr_max_vchans = 24, .clock_autogate_enable = sun6i_enable_clock_autogate_a23, .set_burst_length = sun6i_set_burst_length_a31, + .set_drq = sun6i_set_drq_a31, + .set_mode = sun6i_set_mode_a31, .src_burst_lengths = BIT(1) | BIT(8), .dst_burst_lengths = BIT(1) | BIT(8), .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | @@ -1168,6 +1225,7 @@ static const struct of_device_id sun6i_dma_match[] = { { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg }, { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg }, { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg }, + { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, sun6i_dma_match); @@ -1204,6 +1262,14 @@ static int sun6i_dma_probe(struct platform_device *pdev) return PTR_ERR(sdc->clk); } + if (sdc->cfg->has_mbus_clk) { + sdc->clk_mbus = devm_clk_get(&pdev->dev, "mbus"); + if (IS_ERR(sdc->clk_mbus)) { + dev_err(&pdev->dev, "No mbus clock specified\n"); + return PTR_ERR(sdc->clk_mbus); + } + } + sdc->rstc = devm_reset_control_get(&pdev->dev, NULL); if (IS_ERR(sdc->rstc)) { dev_err(&pdev->dev, "No reset controller specified\n"); @@ -1258,8 +1324,8 @@ static int sun6i_dma_probe(struct platform_device *pdev) ret = of_property_read_u32(np, "dma-requests", &sdc->max_request); if (ret && !sdc->max_request) { dev_info(&pdev->dev, "Missing dma-requests, using %u.\n", - DMA_CHAN_MAX_DRQ); - sdc->max_request = DMA_CHAN_MAX_DRQ; + DMA_CHAN_MAX_DRQ_A31); + sdc->max_request = DMA_CHAN_MAX_DRQ_A31; } /* @@ -1308,11 +1374,19 @@ static int sun6i_dma_probe(struct platform_device *pdev) goto err_reset_assert; } + if (sdc->cfg->has_mbus_clk) { + ret = clk_prepare_enable(sdc->clk_mbus); + if (ret) { + dev_err(&pdev->dev, "Couldn't enable mbus clock\n"); + goto err_clk_disable; + } + } + ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0, dev_name(&pdev->dev), sdc); if (ret) { dev_err(&pdev->dev, "Cannot request IRQ\n"); - goto err_clk_disable; + goto err_mbus_clk_disable; } ret = dma_async_device_register(&sdc->slave); @@ -1337,6 +1411,8 @@ err_dma_unregister: dma_async_device_unregister(&sdc->slave); err_irq_disable: sun6i_kill_tasklet(sdc); +err_mbus_clk_disable: + clk_disable_unprepare(sdc->clk_mbus); err_clk_disable: clk_disable_unprepare(sdc->clk); err_reset_assert: @@ -1355,6 +1431,7 @@ static int sun6i_dma_remove(struct platform_device *pdev) sun6i_kill_tasklet(sdc); + clk_disable_unprepare(sdc->clk_mbus); clk_disable_unprepare(sdc->clk); reset_control_assert(sdc->rstc); diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index ef317c90fbe1..79e9593815f1 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -977,8 +977,12 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; } - if (flags & DMA_PREP_INTERRUPT) + if (flags & DMA_PREP_INTERRUPT) { csr |= TEGRA_APBDMA_CSR_IE_EOC; + } else { + WARN_ON_ONCE(1); + return NULL; + } apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; @@ -1120,8 +1124,12 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; } - if (flags & DMA_PREP_INTERRUPT) + if (flags & DMA_PREP_INTERRUPT) { csr |= TEGRA_APBDMA_CSR_IE_EOC; + } else { + WARN_ON_ONCE(1); + return NULL; + } apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c index bb5390847257..ec4adf4260a0 100644 --- a/drivers/dma/virt-dma.c +++ b/drivers/dma/virt-dma.c @@ -98,7 +98,7 @@ static void vchan_complete(unsigned long arg) } spin_unlock_irq(&vc->lock); - dmaengine_desc_callback_invoke(&cb, NULL); + dmaengine_desc_callback_invoke(&cb, &vd->tx_result); list_for_each_entry_safe(vd, _vd, &head, node) { dmaengine_desc_get_callback(&vd->tx, &cb); @@ -106,7 +106,7 @@ static void vchan_complete(unsigned long arg) list_del(&vd->node); vchan_vdesc_fini(vd); - dmaengine_desc_callback_invoke(&cb, NULL); + dmaengine_desc_callback_invoke(&cb, &vd->tx_result); } } diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index 23342ca23d4a..ab158bac03a7 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -14,6 +14,7 @@ struct virt_dma_desc { struct dma_async_tx_descriptor tx; + struct dmaengine_result tx_result; /* protected by vc.lock */ struct list_head node; }; @@ -62,6 +63,9 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan vd->tx.tx_submit = vchan_tx_submit; vd->tx.desc_free = vchan_tx_desc_free; + vd->tx_result.result = DMA_TRANS_NOERROR; + vd->tx_result.residue = 0; + spin_lock_irqsave(&vc->lock, flags); list_add_tail(&vd->node, &vc->desc_allocated); spin_unlock_irqrestore(&vc->lock, flags); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 36c092349b5b..e7dc3c4dc8e0 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -1095,7 +1095,7 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan) static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) { struct xilinx_vdma_config *config = &chan->config; - struct xilinx_dma_tx_descriptor *desc, *tail_desc; + struct xilinx_dma_tx_descriptor *desc; u32 reg, j; struct xilinx_vdma_tx_segment *segment, *last = NULL; int i = 0; @@ -1112,8 +1112,6 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) desc = list_first_entry(&chan->pending_list, struct xilinx_dma_tx_descriptor, node); - tail_desc = list_last_entry(&chan->pending_list, - struct xilinx_dma_tx_descriptor, node); /* Configure the hardware using info in the config structure */ if (chan->has_vflip) { diff --git a/drivers/firmware/raspberrypi.c b/drivers/firmware/raspberrypi.c index 61be15d9df7d..da26a584dca0 100644 --- a/drivers/firmware/raspberrypi.c +++ b/drivers/firmware/raspberrypi.c @@ -20,6 +20,7 @@ #define MBOX_CHAN_PROPERTY 8 static struct platform_device *rpi_hwmon; +static struct platform_device *rpi_clk; struct rpi_firmware { struct mbox_client cl; @@ -207,6 +208,12 @@ rpi_register_hwmon_driver(struct device *dev, struct rpi_firmware *fw) -1, NULL, 0); } +static void rpi_register_clk_driver(struct device *dev) +{ + rpi_clk = platform_device_register_data(dev, "raspberrypi-clk", + -1, NULL, 0); +} + static int rpi_firmware_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -234,6 +241,7 @@ static int rpi_firmware_probe(struct platform_device *pdev) rpi_firmware_print_firmware_revision(fw); rpi_register_hwmon_driver(dev, fw); + rpi_register_clk_driver(dev); return 0; } @@ -254,6 +262,8 @@ static int rpi_firmware_remove(struct platform_device *pdev) platform_device_unregister(rpi_hwmon); rpi_hwmon = NULL; + platform_device_unregister(rpi_clk); + rpi_clk = NULL; mbox_free_channel(fw->chan); return 0; diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c index ef93406ace1b..7696c692ad5a 100644 --- a/drivers/firmware/ti_sci.c +++ b/drivers/firmware/ti_sci.c @@ -916,7 +916,7 @@ static int ti_sci_cmd_get_device_resets(const struct ti_sci_handle *handle, * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_set_clock_state(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, + u32 dev_id, u32 clk_id, u32 flags, u8 state) { struct ti_sci_info *info; @@ -944,7 +944,12 @@ static int ti_sci_set_clock_state(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_set_clock_state *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } req->request_state = state; ret = ti_sci_do_xfer(info, xfer); @@ -976,7 +981,7 @@ fail: * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_get_clock_state(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, + u32 dev_id, u32 clk_id, u8 *programmed_state, u8 *current_state) { struct ti_sci_info *info; @@ -1007,7 +1012,12 @@ static int ti_sci_cmd_get_clock_state(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_get_clock_state *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } ret = ti_sci_do_xfer(info, xfer); if (ret) { @@ -1047,8 +1057,8 @@ fail: * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_get_clock(const struct ti_sci_handle *handle, u32 dev_id, - u8 clk_id, bool needs_ssc, bool can_change_freq, - bool enable_input_term) + u32 clk_id, bool needs_ssc, + bool can_change_freq, bool enable_input_term) { u32 flags = 0; @@ -1073,7 +1083,7 @@ static int ti_sci_cmd_get_clock(const struct ti_sci_handle *handle, u32 dev_id, * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id) + u32 dev_id, u32 clk_id) { return ti_sci_set_clock_state(handle, dev_id, clk_id, 0, MSG_CLOCK_SW_STATE_UNREQ); @@ -1092,7 +1102,7 @@ static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle, * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_put_clock(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id) + u32 dev_id, u32 clk_id) { return ti_sci_set_clock_state(handle, dev_id, clk_id, 0, MSG_CLOCK_SW_STATE_AUTO); @@ -1110,7 +1120,7 @@ static int ti_sci_cmd_put_clock(const struct ti_sci_handle *handle, * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_is_auto(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, bool *req_state) + u32 dev_id, u32 clk_id, bool *req_state) { u8 state = 0; int ret; @@ -1139,7 +1149,7 @@ static int ti_sci_cmd_clk_is_auto(const struct ti_sci_handle *handle, * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_is_on(const struct ti_sci_handle *handle, u32 dev_id, - u8 clk_id, bool *req_state, bool *curr_state) + u32 clk_id, bool *req_state, bool *curr_state) { u8 c_state = 0, r_state = 0; int ret; @@ -1172,7 +1182,7 @@ static int ti_sci_cmd_clk_is_on(const struct ti_sci_handle *handle, u32 dev_id, * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_is_off(const struct ti_sci_handle *handle, u32 dev_id, - u8 clk_id, bool *req_state, bool *curr_state) + u32 clk_id, bool *req_state, bool *curr_state) { u8 c_state = 0, r_state = 0; int ret; @@ -1204,7 +1214,7 @@ static int ti_sci_cmd_clk_is_off(const struct ti_sci_handle *handle, u32 dev_id, * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_set_parent(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, u8 parent_id) + u32 dev_id, u32 clk_id, u32 parent_id) { struct ti_sci_info *info; struct ti_sci_msg_req_set_clock_parent *req; @@ -1231,8 +1241,18 @@ static int ti_sci_cmd_clk_set_parent(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_set_clock_parent *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; - req->parent_id = parent_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } + if (parent_id < 255) { + req->parent_id = parent_id; + } else { + req->parent_id = 255; + req->parent_id_32 = parent_id; + } ret = ti_sci_do_xfer(info, xfer); if (ret) { @@ -1262,7 +1282,7 @@ fail: * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, u8 *parent_id) + u32 dev_id, u32 clk_id, u32 *parent_id) { struct ti_sci_info *info; struct ti_sci_msg_req_get_clock_parent *req; @@ -1289,7 +1309,12 @@ static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_get_clock_parent *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } ret = ti_sci_do_xfer(info, xfer); if (ret) { @@ -1299,10 +1324,14 @@ static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle, resp = (struct ti_sci_msg_resp_get_clock_parent *)xfer->xfer_buf; - if (!ti_sci_is_response_ack(resp)) + if (!ti_sci_is_response_ack(resp)) { ret = -ENODEV; - else - *parent_id = resp->parent_id; + } else { + if (resp->parent_id < 255) + *parent_id = resp->parent_id; + else + *parent_id = resp->parent_id_32; + } fail: ti_sci_put_one_xfer(&info->minfo, xfer); @@ -1322,8 +1351,8 @@ fail: * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, - u8 *num_parents) + u32 dev_id, u32 clk_id, + u32 *num_parents) { struct ti_sci_info *info; struct ti_sci_msg_req_get_clock_num_parents *req; @@ -1350,7 +1379,12 @@ static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_get_clock_num_parents *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } ret = ti_sci_do_xfer(info, xfer); if (ret) { @@ -1360,10 +1394,14 @@ static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle, resp = (struct ti_sci_msg_resp_get_clock_num_parents *)xfer->xfer_buf; - if (!ti_sci_is_response_ack(resp)) + if (!ti_sci_is_response_ack(resp)) { ret = -ENODEV; - else - *num_parents = resp->num_parents; + } else { + if (resp->num_parents < 255) + *num_parents = resp->num_parents; + else + *num_parents = resp->num_parents_32; + } fail: ti_sci_put_one_xfer(&info->minfo, xfer); @@ -1391,7 +1429,7 @@ fail: * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_get_match_freq(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, u64 min_freq, + u32 dev_id, u32 clk_id, u64 min_freq, u64 target_freq, u64 max_freq, u64 *match_freq) { @@ -1420,7 +1458,12 @@ static int ti_sci_cmd_clk_get_match_freq(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_query_clock_freq *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } req->min_freq_hz = min_freq; req->target_freq_hz = target_freq; req->max_freq_hz = max_freq; @@ -1463,7 +1506,7 @@ fail: * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_set_freq(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, u64 min_freq, + u32 dev_id, u32 clk_id, u64 min_freq, u64 target_freq, u64 max_freq) { struct ti_sci_info *info; @@ -1491,7 +1534,12 @@ static int ti_sci_cmd_clk_set_freq(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_set_clock_freq *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } req->min_freq_hz = min_freq; req->target_freq_hz = target_freq; req->max_freq_hz = max_freq; @@ -1524,7 +1572,7 @@ fail: * Return: 0 if all went well, else returns appropriate error value. */ static int ti_sci_cmd_clk_get_freq(const struct ti_sci_handle *handle, - u32 dev_id, u8 clk_id, u64 *freq) + u32 dev_id, u32 clk_id, u64 *freq) { struct ti_sci_info *info; struct ti_sci_msg_req_get_clock_freq *req; @@ -1551,7 +1599,12 @@ static int ti_sci_cmd_clk_get_freq(const struct ti_sci_handle *handle, } req = (struct ti_sci_msg_req_get_clock_freq *)xfer->xfer_buf; req->dev_id = dev_id; - req->clk_id = clk_id; + if (clk_id < 255) { + req->clk_id = clk_id; + } else { + req->clk_id = 255; + req->clk_id_32 = clk_id; + } ret = ti_sci_do_xfer(info, xfer); if (ret) { @@ -2349,12 +2402,13 @@ devm_ti_sci_get_of_resource(const struct ti_sci_handle *handle, if (!res) return ERR_PTR(-ENOMEM); - res->sets = of_property_count_elems_of_size(dev_of_node(dev), of_prop, - sizeof(u32)); - if (res->sets < 0) { + ret = of_property_count_elems_of_size(dev_of_node(dev), of_prop, + sizeof(u32)); + if (ret < 0) { dev_err(dev, "%s resource type ids not available\n", of_prop); - return ERR_PTR(res->sets); + return ERR_PTR(ret); } + res->sets = ret; res->desc = devm_kcalloc(dev, res->sets, sizeof(*res->desc), GFP_KERNEL); diff --git a/drivers/firmware/ti_sci.h b/drivers/firmware/ti_sci.h index adbeeefaca92..414e0ced5409 100644 --- a/drivers/firmware/ti_sci.h +++ b/drivers/firmware/ti_sci.h @@ -202,7 +202,8 @@ struct ti_sci_msg_req_set_device_resets { * @dev_id: Device identifier this request is for * @clk_id: Clock identifier for the device for this request. * Each device has it's own set of clock inputs. This indexes - * which clock input to modify. + * which clock input to modify. Set to 255 if clock ID is + * greater than or equal to 255. * @request_state: Request the state for the clock to be set to. * MSG_CLOCK_SW_STATE_UNREQ: The IP does not require this clock, * it can be disabled, regardless of the state of the device @@ -213,6 +214,9 @@ struct ti_sci_msg_req_set_device_resets { * being required by the device.(default) * MSG_CLOCK_SW_STATE_REQ: Configure the clock to be enabled, * regardless of the state of the device. + * @clk_id_32: Clock identifier for the device for this request. + * Only to be used if the clock ID is greater than or equal to + * 255. * * Normally, all required clocks are managed by TISCI entity, this is used * only for specific control *IF* required. Auto managed state is @@ -234,6 +238,7 @@ struct ti_sci_msg_req_set_clock_state { #define MSG_CLOCK_SW_STATE_AUTO 1 #define MSG_CLOCK_SW_STATE_REQ 2 u8 request_state; + u32 clk_id_32; } __packed; /** @@ -242,7 +247,11 @@ struct ti_sci_msg_req_set_clock_state { * @dev_id: Device identifier this request is for * @clk_id: Clock identifier for the device for this request. * Each device has it's own set of clock inputs. This indexes - * which clock input to get state of. + * which clock input to get state of. Set to 255 if the clock + * ID is greater than or equal to 255. + * @clk_id_32: Clock identifier for the device for the request. + * Only to be used if the clock ID is greater than or equal to + * 255. * * Request type is TI_SCI_MSG_GET_CLOCK_STATE, response is state * of the clock @@ -251,6 +260,7 @@ struct ti_sci_msg_req_get_clock_state { struct ti_sci_msg_hdr hdr; u32 dev_id; u8 clk_id; + u32 clk_id_32; } __packed; /** @@ -278,9 +288,13 @@ struct ti_sci_msg_resp_get_clock_state { * @dev_id: Device identifier this request is for * @clk_id: Clock identifier for the device for this request. * Each device has it's own set of clock inputs. This indexes - * which clock input to modify. + * which clock input to modify. Set to 255 if clock ID is + * greater than or equal to 255. * @parent_id: The new clock parent is selectable by an index via this - * parameter. + * parameter. Set to 255 if clock ID is greater than or + * equal to 255. + * @clk_id_32: Clock identifier if @clk_id field is 255. + * @parent_id_32: Parent identifier if @parent_id is 255. * * Request type is TI_SCI_MSG_SET_CLOCK_PARENT, response is generic * ACK / NACK message. @@ -290,6 +304,8 @@ struct ti_sci_msg_req_set_clock_parent { u32 dev_id; u8 clk_id; u8 parent_id; + u32 clk_id_32; + u32 parent_id_32; } __packed; /** @@ -298,7 +314,10 @@ struct ti_sci_msg_req_set_clock_parent { * @dev_id: Device identifier this request is for * @clk_id: Clock identifier for the device for this request. * Each device has it's own set of clock inputs. This indexes - * which clock input to get the parent for. + * which clock input to get the parent for. If this field + * contains 255, the actual clock identifier is stored in + * @clk_id_32. + * @clk_id_32: Clock identifier if the @clk_id field contains 255. * * Request type is TI_SCI_MSG_GET_CLOCK_PARENT, response is parent information */ @@ -306,25 +325,32 @@ struct ti_sci_msg_req_get_clock_parent { struct ti_sci_msg_hdr hdr; u32 dev_id; u8 clk_id; + u32 clk_id_32; } __packed; /** * struct ti_sci_msg_resp_get_clock_parent - Response with clock parent * @hdr: Generic Header - * @parent_id: The current clock parent + * @parent_id: The current clock parent. If set to 255, the current parent + * ID can be found from the @parent_id_32 field. + * @parent_id_32: Current clock parent if @parent_id field is set to + * 255. * * Response to TI_SCI_MSG_GET_CLOCK_PARENT. */ struct ti_sci_msg_resp_get_clock_parent { struct ti_sci_msg_hdr hdr; u8 parent_id; + u32 parent_id_32; } __packed; /** * struct ti_sci_msg_req_get_clock_num_parents - Request to get clock parents * @hdr: Generic header * @dev_id: Device identifier this request is for - * @clk_id: Clock identifier for the device for this request. + * @clk_id: Clock identifier for the device for this request. Set to + * 255 if clock ID is greater than or equal to 255. + * @clk_id_32: Clock identifier if the @clk_id field contains 255. * * This request provides information about how many clock parent options * are available for a given clock to a device. This is typically used @@ -337,18 +363,24 @@ struct ti_sci_msg_req_get_clock_num_parents { struct ti_sci_msg_hdr hdr; u32 dev_id; u8 clk_id; + u32 clk_id_32; } __packed; /** * struct ti_sci_msg_resp_get_clock_num_parents - Response for get clk parents * @hdr: Generic header - * @num_parents: Number of clock parents + * @num_parents: Number of clock parents. If set to 255, the actual + * number of parents is stored into @num_parents_32 + * field instead. + * @num_parents_32: Number of clock parents if @num_parents field is + * set to 255. * * Response to TI_SCI_MSG_GET_NUM_CLOCK_PARENTS */ struct ti_sci_msg_resp_get_clock_num_parents { struct ti_sci_msg_hdr hdr; u8 num_parents; + u32 num_parents_32; } __packed; /** @@ -363,7 +395,9 @@ struct ti_sci_msg_resp_get_clock_num_parents { * @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum * allowable programmed frequency and does not account for clock * tolerances and jitter. - * @clk_id: Clock identifier for the device for this request. + * @clk_id: Clock identifier for the device for this request. Set to + * 255 if clock identifier is greater than or equal to 255. + * @clk_id_32: Clock identifier if @clk_id is set to 255. * * NOTE: Normally clock frequency management is automatically done by TISCI * entity. In case of specific requests, TISCI evaluates capability to achieve @@ -380,6 +414,7 @@ struct ti_sci_msg_req_query_clock_freq { u64 target_freq_hz; u64 max_freq_hz; u8 clk_id; + u32 clk_id_32; } __packed; /** @@ -407,7 +442,9 @@ struct ti_sci_msg_resp_query_clock_freq { * @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum * allowable programmed frequency and does not account for clock * tolerances and jitter. - * @clk_id: Clock identifier for the device for this request. + * @clk_id: Clock identifier for the device for this request. Set to + * 255 if clock ID is greater than or equal to 255. + * @clk_id_32: Clock identifier if @clk_id field is set to 255. * * NOTE: Normally clock frequency management is automatically done by TISCI * entity. In case of specific requests, TISCI evaluates capability to achieve @@ -436,13 +473,16 @@ struct ti_sci_msg_req_set_clock_freq { u64 target_freq_hz; u64 max_freq_hz; u8 clk_id; + u32 clk_id_32; } __packed; /** * struct ti_sci_msg_req_get_clock_freq - Request to get the clock frequency * @hdr: Generic Header * @dev_id: Device identifier this request is for - * @clk_id: Clock identifier for the device for this request. + * @clk_id: Clock identifier for the device for this request. Set to + * 255 if clock ID is greater than or equal to 255. + * @clk_id_32: Clock identifier if @clk_id field is set to 255. * * NOTE: Normally clock frequency management is automatically done by TISCI * entity. In some cases, clock frequencies are configured by host. @@ -454,6 +494,7 @@ struct ti_sci_msg_req_get_clock_freq { struct ti_sci_msg_hdr hdr; u32 dev_id; u8 clk_id; + u32 clk_id_32; } __packed; /** diff --git a/drivers/fpga/dfl-afu-dma-region.c b/drivers/fpga/dfl-afu-dma-region.c index dcd80b088c7b..62f924489db5 100644 --- a/drivers/fpga/dfl-afu-dma-region.c +++ b/drivers/fpga/dfl-afu-dma-region.c @@ -12,6 +12,7 @@ #include <linux/dma-mapping.h> #include <linux/sched/signal.h> #include <linux/uaccess.h> +#include <linux/mm.h> #include "dfl-afu.h" @@ -32,52 +33,6 @@ void afu_dma_region_init(struct dfl_feature_platform_data *pdata) } /** - * afu_dma_adjust_locked_vm - adjust locked memory - * @dev: port device - * @npages: number of pages - * @incr: increase or decrease locked memory - * - * Increase or decrease the locked memory size with npages input. - * - * Return 0 on success. - * Return -ENOMEM if locked memory size is over the limit and no CAP_IPC_LOCK. - */ -static int afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr) -{ - unsigned long locked, lock_limit; - int ret = 0; - - /* the task is exiting. */ - if (!current->mm) - return 0; - - down_write(¤t->mm->mmap_sem); - - if (incr) { - locked = current->mm->locked_vm + npages; - lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - - if (locked > lock_limit && !capable(CAP_IPC_LOCK)) - ret = -ENOMEM; - else - current->mm->locked_vm += npages; - } else { - if (WARN_ON_ONCE(npages > current->mm->locked_vm)) - npages = current->mm->locked_vm; - current->mm->locked_vm -= npages; - } - - dev_dbg(dev, "[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n", current->pid, - incr ? '+' : '-', npages << PAGE_SHIFT, - current->mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK), - ret ? "- exceeded" : ""); - - up_write(¤t->mm->mmap_sem); - - return ret; -} - -/** * afu_dma_pin_pages - pin pages of given dma memory region * @pdata: feature device platform data * @region: dma memory region to be pinned @@ -92,7 +47,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata, struct device *dev = &pdata->dev->dev; int ret, pinned; - ret = afu_dma_adjust_locked_vm(dev, npages, true); + ret = account_locked_vm(current->mm, npages, true); if (ret) return ret; @@ -121,7 +76,7 @@ put_pages: free_pages: kfree(region->pages); unlock_vm: - afu_dma_adjust_locked_vm(dev, npages, false); + account_locked_vm(current->mm, npages, false); return ret; } @@ -141,7 +96,7 @@ static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata, put_all_pages(region->pages, npages); kfree(region->pages); - afu_dma_adjust_locked_vm(dev, npages, false); + account_locked_vm(current->mm, npages, false); dev_dbg(dev, "%ld pages unpinned\n", npages); } diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c index fc494a84a29d..e0b025689625 100644 --- a/drivers/gpio/gpio-davinci.c +++ b/drivers/gpio/gpio-davinci.c @@ -238,8 +238,9 @@ static int davinci_gpio_probe(struct platform_device *pdev) for (i = 0; i < nirq; i++) { chips->irqs[i] = platform_get_irq(pdev, i); if (chips->irqs[i] < 0) { - dev_info(dev, "IRQ not populated, err = %d\n", - chips->irqs[i]); + if (chips->irqs[i] != -EPROBE_DEFER) + dev_info(dev, "IRQ not populated, err = %d\n", + chips->irqs[i]); return chips->irqs[i]; } } diff --git a/drivers/gpio/gpio-em.c b/drivers/gpio/gpio-em.c index b6af705a4e5f..a87951293aaa 100644 --- a/drivers/gpio/gpio-em.c +++ b/drivers/gpio/gpio-em.c @@ -259,6 +259,13 @@ static const struct irq_domain_ops em_gio_irq_domain_ops = { .xlate = irq_domain_xlate_twocell, }; +static void em_gio_irq_domain_remove(void *data) +{ + struct irq_domain *domain = data; + + irq_domain_remove(domain); +} + static int em_gio_probe(struct platform_device *pdev) { struct em_gio_priv *p; @@ -333,39 +340,30 @@ static int em_gio_probe(struct platform_device *pdev) return -ENXIO; } + ret = devm_add_action_or_reset(&pdev->dev, em_gio_irq_domain_remove, + p->irq_domain); + if (ret) + return ret; + if (devm_request_irq(&pdev->dev, irq[0]->start, em_gio_irq_handler, 0, name, p)) { dev_err(&pdev->dev, "failed to request low IRQ\n"); - ret = -ENOENT; - goto err1; + return -ENOENT; } if (devm_request_irq(&pdev->dev, irq[1]->start, em_gio_irq_handler, 0, name, p)) { dev_err(&pdev->dev, "failed to request high IRQ\n"); - ret = -ENOENT; - goto err1; + return -ENOENT; } ret = devm_gpiochip_add_data(&pdev->dev, gpio_chip, p); if (ret) { dev_err(&pdev->dev, "failed to add GPIO controller\n"); - goto err1; + return ret; } return 0; - -err1: - irq_domain_remove(p->irq_domain); - return ret; -} - -static int em_gio_remove(struct platform_device *pdev) -{ - struct em_gio_priv *p = platform_get_drvdata(pdev); - - irq_domain_remove(p->irq_domain); - return 0; } static const struct of_device_id em_gio_dt_ids[] = { @@ -376,7 +374,6 @@ MODULE_DEVICE_TABLE(of, em_gio_dt_ids); static struct platform_driver em_gio_device_driver = { .probe = em_gio_probe, - .remove = em_gio_remove, .driver = { .name = "em_gio", .of_match_table = em_gio_dt_ids, diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index f974075ff00e..567fb98c0892 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -118,15 +118,8 @@ static void of_gpio_flags_quirks(struct device_node *np, * Legacy handling of SPI active high chip select. If we have a * property named "cs-gpios" we need to inspect the child node * to determine if the flags should have inverted semantics. - * - * This does not apply to an SPI device named "spi-gpio", because - * these have traditionally obtained their own GPIOs by parsing - * the device tree directly and did not respect any "spi-cs-high" - * property on the SPI bus children. */ - if (IS_ENABLED(CONFIG_SPI_MASTER) && - !strcmp(propname, "cs-gpios") && - !of_device_is_compatible(np, "spi-gpio") && + if (IS_ENABLED(CONFIG_SPI_MASTER) && !strcmp(propname, "cs-gpios") && of_property_read_bool(np, "cs-gpios")) { struct device_node *child; u32 cs; @@ -161,6 +154,7 @@ static void of_gpio_flags_quirks(struct device_node *np, of_node_full_name(child)); *flags |= OF_GPIO_ACTIVE_LOW; } + of_node_put(child); break; } } diff --git a/drivers/hwspinlock/Kconfig b/drivers/hwspinlock/Kconfig index 7869c67e5b6b..37740e992cfa 100644 --- a/drivers/hwspinlock/Kconfig +++ b/drivers/hwspinlock/Kconfig @@ -9,7 +9,7 @@ menuconfig HWSPINLOCK config HWSPINLOCK_OMAP tristate "OMAP Hardware Spinlock device" depends on HWSPINLOCK - depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX || SOC_AM33XX || SOC_AM43XX + depends on ARCH_OMAP4 || SOC_OMAP5 || SOC_DRA7XX || SOC_AM33XX || SOC_AM43XX || ARCH_K3 help Say y here to support the OMAP Hardware Spinlock device (firstly introduced in OMAP4). diff --git a/drivers/hwspinlock/hwspinlock_core.c b/drivers/hwspinlock/hwspinlock_core.c index 2bad40d42210..8862445aa858 100644 --- a/drivers/hwspinlock/hwspinlock_core.c +++ b/drivers/hwspinlock/hwspinlock_core.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) "%s: " fmt, __func__ +#include <linux/delay.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/spinlock.h> @@ -23,6 +24,9 @@ #include "hwspinlock_internal.h" +/* retry delay used in atomic context */ +#define HWSPINLOCK_RETRY_DELAY_US 100 + /* radix tree tags */ #define HWSPINLOCK_UNUSED (0) /* tags an hwspinlock as unused */ @@ -68,11 +72,11 @@ static DEFINE_MUTEX(hwspinlock_tree_lock); * user need some time-consuming or sleepable operations under the hardware * lock, they need one sleepable lock (like mutex) to protect the operations. * - * If the mode is not HWLOCK_RAW, upon a successful return from this function, - * preemption (and possibly interrupts) is disabled, so the caller must not - * sleep, and is advised to release the hwspinlock as soon as possible. This is - * required in order to minimize remote cores polling on the hardware - * interconnect. + * If the mode is neither HWLOCK_IN_ATOMIC nor HWLOCK_RAW, upon a successful + * return from this function, preemption (and possibly interrupts) is disabled, + * so the caller must not sleep, and is advised to release the hwspinlock as + * soon as possible. This is required in order to minimize remote cores polling + * on the hardware interconnect. * * The user decides whether local interrupts are disabled or not, and if yes, * whether he wants their previous state to be saved. It is up to the user @@ -112,6 +116,7 @@ int __hwspin_trylock(struct hwspinlock *hwlock, int mode, unsigned long *flags) ret = spin_trylock_irq(&hwlock->lock); break; case HWLOCK_RAW: + case HWLOCK_IN_ATOMIC: ret = 1; break; default: @@ -136,6 +141,7 @@ int __hwspin_trylock(struct hwspinlock *hwlock, int mode, unsigned long *flags) spin_unlock_irq(&hwlock->lock); break; case HWLOCK_RAW: + case HWLOCK_IN_ATOMIC: /* Nothing to do */ break; default: @@ -179,11 +185,14 @@ EXPORT_SYMBOL_GPL(__hwspin_trylock); * user need some time-consuming or sleepable operations under the hardware * lock, they need one sleepable lock (like mutex) to protect the operations. * - * If the mode is not HWLOCK_RAW, upon a successful return from this function, - * preemption is disabled (and possibly local interrupts, too), so the caller - * must not sleep, and is advised to release the hwspinlock as soon as possible. - * This is required in order to minimize remote cores polling on the - * hardware interconnect. + * If the mode is HWLOCK_IN_ATOMIC (called from an atomic context) the timeout + * is handled with busy-waiting delays, hence shall not exceed few msecs. + * + * If the mode is neither HWLOCK_IN_ATOMIC nor HWLOCK_RAW, upon a successful + * return from this function, preemption (and possibly interrupts) is disabled, + * so the caller must not sleep, and is advised to release the hwspinlock as + * soon as possible. This is required in order to minimize remote cores polling + * on the hardware interconnect. * * The user decides whether local interrupts are disabled or not, and if yes, * whether he wants their previous state to be saved. It is up to the user @@ -198,7 +207,7 @@ int __hwspin_lock_timeout(struct hwspinlock *hwlock, unsigned int to, int mode, unsigned long *flags) { int ret; - unsigned long expire; + unsigned long expire, atomic_delay = 0; expire = msecs_to_jiffies(to) + jiffies; @@ -212,8 +221,15 @@ int __hwspin_lock_timeout(struct hwspinlock *hwlock, unsigned int to, * The lock is already taken, let's check if the user wants * us to try again */ - if (time_is_before_eq_jiffies(expire)) - return -ETIMEDOUT; + if (mode == HWLOCK_IN_ATOMIC) { + udelay(HWSPINLOCK_RETRY_DELAY_US); + atomic_delay += HWSPINLOCK_RETRY_DELAY_US; + if (atomic_delay > to * 1000) + return -ETIMEDOUT; + } else { + if (time_is_before_eq_jiffies(expire)) + return -ETIMEDOUT; + } /* * Allow platform-specific relax handlers to prevent @@ -276,6 +292,7 @@ void __hwspin_unlock(struct hwspinlock *hwlock, int mode, unsigned long *flags) spin_unlock_irq(&hwlock->lock); break; case HWLOCK_RAW: + case HWLOCK_IN_ATOMIC: /* Nothing to do */ break; default: @@ -333,6 +350,11 @@ int of_hwspin_lock_get_id(struct device_node *np, int index) if (ret) return ret; + if (!of_device_is_available(args.np)) { + ret = -ENOENT; + goto out; + } + /* Find the hwspinlock device: we need its base_id */ ret = -EPROBE_DEFER; rcu_read_lock(); diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 625844e0abef..14e1a532ebb5 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -140,6 +140,9 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) if (ret) goto reg_fail; + dev_dbg(&pdev->dev, "Registered %d locks with HwSpinlock core\n", + num_locks); + return 0; reg_fail: @@ -171,6 +174,7 @@ static int omap_hwspinlock_remove(struct platform_device *pdev) static const struct of_device_id omap_hwspinlock_of_match[] = { { .compatible = "ti,omap4-hwspinlock", }, + { .compatible = "ti,am654-hwspinlock", }, { /* end */ }, }; MODULE_DEVICE_TABLE(of, omap_hwspinlock_of_match); diff --git a/drivers/hwspinlock/stm32_hwspinlock.c b/drivers/hwspinlock/stm32_hwspinlock.c index 441839288893..c8eacf4f9692 100644 --- a/drivers/hwspinlock/stm32_hwspinlock.c +++ b/drivers/hwspinlock/stm32_hwspinlock.c @@ -5,6 +5,7 @@ */ #include <linux/clk.h> +#include <linux/delay.h> #include <linux/hwspinlock.h> #include <linux/io.h> #include <linux/kernel.h> @@ -42,9 +43,15 @@ static void stm32_hwspinlock_unlock(struct hwspinlock *lock) writel(STM32_MUTEX_COREID, lock_addr); } +static void stm32_hwspinlock_relax(struct hwspinlock *lock) +{ + ndelay(50); +} + static const struct hwspinlock_ops stm32_hwspinlock_ops = { .trylock = stm32_hwspinlock_trylock, .unlock = stm32_hwspinlock_unlock, + .relax = stm32_hwspinlock_relax, }; static int stm32_hwspinlock_probe(struct platform_device *pdev) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 83664db5221d..e15cdcd8cb3c 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -473,4 +473,15 @@ config HYPERV_IOMMU Stub IOMMU driver to handle IRQs as to allow Hyper-V Linux guests to run with x2APIC mode enabled. +config VIRTIO_IOMMU + bool "Virtio IOMMU driver" + depends on VIRTIO=y + depends on ARM64 + select IOMMU_API + select INTERVAL_TREE + help + Para-virtualised IOMMU driver with virtio. + + Say Y here if you intend to run this kernel as a guest. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 8c71a15e986b..f13f36ae1af6 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -33,3 +33,4 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o +obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c new file mode 100644 index 000000000000..433f4d2ee956 --- /dev/null +++ b/drivers/iommu/virtio-iommu.c @@ -0,0 +1,1158 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Virtio driver for the paravirtualized IOMMU + * + * Copyright (C) 2018 Arm Limited + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/amba/bus.h> +#include <linux/delay.h> +#include <linux/dma-iommu.h> +#include <linux/freezer.h> +#include <linux/interval_tree.h> +#include <linux/iommu.h> +#include <linux/module.h> +#include <linux/of_iommu.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/virtio.h> +#include <linux/virtio_config.h> +#include <linux/virtio_ids.h> +#include <linux/wait.h> + +#include <uapi/linux/virtio_iommu.h> + +#define MSI_IOVA_BASE 0x8000000 +#define MSI_IOVA_LENGTH 0x100000 + +#define VIOMMU_REQUEST_VQ 0 +#define VIOMMU_EVENT_VQ 1 +#define VIOMMU_NR_VQS 2 + +struct viommu_dev { + struct iommu_device iommu; + struct device *dev; + struct virtio_device *vdev; + + struct ida domain_ids; + + struct virtqueue *vqs[VIOMMU_NR_VQS]; + spinlock_t request_lock; + struct list_head requests; + void *evts; + + /* Device configuration */ + struct iommu_domain_geometry geometry; + u64 pgsize_bitmap; + u8 domain_bits; + u32 probe_size; +}; + +struct viommu_mapping { + phys_addr_t paddr; + struct interval_tree_node iova; + u32 flags; +}; + +struct viommu_domain { + struct iommu_domain domain; + struct viommu_dev *viommu; + struct mutex mutex; /* protects viommu pointer */ + unsigned int id; + + spinlock_t mappings_lock; + struct rb_root_cached mappings; + + unsigned long nr_endpoints; +}; + +struct viommu_endpoint { + struct device *dev; + struct viommu_dev *viommu; + struct viommu_domain *vdomain; + struct list_head resv_regions; +}; + +struct viommu_request { + struct list_head list; + void *writeback; + unsigned int write_offset; + unsigned int len; + char buf[]; +}; + +#define VIOMMU_FAULT_RESV_MASK 0xffffff00 + +struct viommu_event { + union { + u32 head; + struct virtio_iommu_fault fault; + }; +}; + +#define to_viommu_domain(domain) \ + container_of(domain, struct viommu_domain, domain) + +static int viommu_get_req_errno(void *buf, size_t len) +{ + struct virtio_iommu_req_tail *tail = buf + len - sizeof(*tail); + + switch (tail->status) { + case VIRTIO_IOMMU_S_OK: + return 0; + case VIRTIO_IOMMU_S_UNSUPP: + return -ENOSYS; + case VIRTIO_IOMMU_S_INVAL: + return -EINVAL; + case VIRTIO_IOMMU_S_RANGE: + return -ERANGE; + case VIRTIO_IOMMU_S_NOENT: + return -ENOENT; + case VIRTIO_IOMMU_S_FAULT: + return -EFAULT; + case VIRTIO_IOMMU_S_IOERR: + case VIRTIO_IOMMU_S_DEVERR: + default: + return -EIO; + } +} + +static void viommu_set_req_status(void *buf, size_t len, int status) +{ + struct virtio_iommu_req_tail *tail = buf + len - sizeof(*tail); + + tail->status = status; +} + +static off_t viommu_get_write_desc_offset(struct viommu_dev *viommu, + struct virtio_iommu_req_head *req, + size_t len) +{ + size_t tail_size = sizeof(struct virtio_iommu_req_tail); + + if (req->type == VIRTIO_IOMMU_T_PROBE) + return len - viommu->probe_size - tail_size; + + return len - tail_size; +} + +/* + * __viommu_sync_req - Complete all in-flight requests + * + * Wait for all added requests to complete. When this function returns, all + * requests that were in-flight at the time of the call have completed. + */ +static int __viommu_sync_req(struct viommu_dev *viommu) +{ + int ret = 0; + unsigned int len; + size_t write_len; + struct viommu_request *req; + struct virtqueue *vq = viommu->vqs[VIOMMU_REQUEST_VQ]; + + assert_spin_locked(&viommu->request_lock); + + virtqueue_kick(vq); + + while (!list_empty(&viommu->requests)) { + len = 0; + req = virtqueue_get_buf(vq, &len); + if (!req) + continue; + + if (!len) + viommu_set_req_status(req->buf, req->len, + VIRTIO_IOMMU_S_IOERR); + + write_len = req->len - req->write_offset; + if (req->writeback && len == write_len) + memcpy(req->writeback, req->buf + req->write_offset, + write_len); + + list_del(&req->list); + kfree(req); + } + + return ret; +} + +static int viommu_sync_req(struct viommu_dev *viommu) +{ + int ret; + unsigned long flags; + + spin_lock_irqsave(&viommu->request_lock, flags); + ret = __viommu_sync_req(viommu); + if (ret) + dev_dbg(viommu->dev, "could not sync requests (%d)\n", ret); + spin_unlock_irqrestore(&viommu->request_lock, flags); + + return ret; +} + +/* + * __viommu_add_request - Add one request to the queue + * @buf: pointer to the request buffer + * @len: length of the request buffer + * @writeback: copy data back to the buffer when the request completes. + * + * Add a request to the queue. Only synchronize the queue if it's already full. + * Otherwise don't kick the queue nor wait for requests to complete. + * + * When @writeback is true, data written by the device, including the request + * status, is copied into @buf after the request completes. This is unsafe if + * the caller allocates @buf on stack and drops the lock between add_req() and + * sync_req(). + * + * Return 0 if the request was successfully added to the queue. + */ +static int __viommu_add_req(struct viommu_dev *viommu, void *buf, size_t len, + bool writeback) +{ + int ret; + off_t write_offset; + struct viommu_request *req; + struct scatterlist top_sg, bottom_sg; + struct scatterlist *sg[2] = { &top_sg, &bottom_sg }; + struct virtqueue *vq = viommu->vqs[VIOMMU_REQUEST_VQ]; + + assert_spin_locked(&viommu->request_lock); + + write_offset = viommu_get_write_desc_offset(viommu, buf, len); + if (write_offset <= 0) + return -EINVAL; + + req = kzalloc(sizeof(*req) + len, GFP_ATOMIC); + if (!req) + return -ENOMEM; + + req->len = len; + if (writeback) { + req->writeback = buf + write_offset; + req->write_offset = write_offset; + } + memcpy(&req->buf, buf, write_offset); + + sg_init_one(&top_sg, req->buf, write_offset); + sg_init_one(&bottom_sg, req->buf + write_offset, len - write_offset); + + ret = virtqueue_add_sgs(vq, sg, 1, 1, req, GFP_ATOMIC); + if (ret == -ENOSPC) { + /* If the queue is full, sync and retry */ + if (!__viommu_sync_req(viommu)) + ret = virtqueue_add_sgs(vq, sg, 1, 1, req, GFP_ATOMIC); + } + if (ret) + goto err_free; + + list_add_tail(&req->list, &viommu->requests); + return 0; + +err_free: + kfree(req); + return ret; +} + +static int viommu_add_req(struct viommu_dev *viommu, void *buf, size_t len) +{ + int ret; + unsigned long flags; + + spin_lock_irqsave(&viommu->request_lock, flags); + ret = __viommu_add_req(viommu, buf, len, false); + if (ret) + dev_dbg(viommu->dev, "could not add request: %d\n", ret); + spin_unlock_irqrestore(&viommu->request_lock, flags); + + return ret; +} + +/* + * Send a request and wait for it to complete. Return the request status (as an + * errno) + */ +static int viommu_send_req_sync(struct viommu_dev *viommu, void *buf, + size_t len) +{ + int ret; + unsigned long flags; + + spin_lock_irqsave(&viommu->request_lock, flags); + + ret = __viommu_add_req(viommu, buf, len, true); + if (ret) { + dev_dbg(viommu->dev, "could not add request (%d)\n", ret); + goto out_unlock; + } + + ret = __viommu_sync_req(viommu); + if (ret) { + dev_dbg(viommu->dev, "could not sync requests (%d)\n", ret); + /* Fall-through (get the actual request status) */ + } + + ret = viommu_get_req_errno(buf, len); +out_unlock: + spin_unlock_irqrestore(&viommu->request_lock, flags); + return ret; +} + +/* + * viommu_add_mapping - add a mapping to the internal tree + * + * On success, return the new mapping. Otherwise return NULL. + */ +static int viommu_add_mapping(struct viommu_domain *vdomain, unsigned long iova, + phys_addr_t paddr, size_t size, u32 flags) +{ + unsigned long irqflags; + struct viommu_mapping *mapping; + + mapping = kzalloc(sizeof(*mapping), GFP_ATOMIC); + if (!mapping) + return -ENOMEM; + + mapping->paddr = paddr; + mapping->iova.start = iova; + mapping->iova.last = iova + size - 1; + mapping->flags = flags; + + spin_lock_irqsave(&vdomain->mappings_lock, irqflags); + interval_tree_insert(&mapping->iova, &vdomain->mappings); + spin_unlock_irqrestore(&vdomain->mappings_lock, irqflags); + + return 0; +} + +/* + * viommu_del_mappings - remove mappings from the internal tree + * + * @vdomain: the domain + * @iova: start of the range + * @size: size of the range. A size of 0 corresponds to the entire address + * space. + * + * On success, returns the number of unmapped bytes (>= size) + */ +static size_t viommu_del_mappings(struct viommu_domain *vdomain, + unsigned long iova, size_t size) +{ + size_t unmapped = 0; + unsigned long flags; + unsigned long last = iova + size - 1; + struct viommu_mapping *mapping = NULL; + struct interval_tree_node *node, *next; + + spin_lock_irqsave(&vdomain->mappings_lock, flags); + next = interval_tree_iter_first(&vdomain->mappings, iova, last); + while (next) { + node = next; + mapping = container_of(node, struct viommu_mapping, iova); + next = interval_tree_iter_next(node, iova, last); + + /* Trying to split a mapping? */ + if (mapping->iova.start < iova) + break; + + /* + * Virtio-iommu doesn't allow UNMAP to split a mapping created + * with a single MAP request, so remove the full mapping. + */ + unmapped += mapping->iova.last - mapping->iova.start + 1; + + interval_tree_remove(node, &vdomain->mappings); + kfree(mapping); + } + spin_unlock_irqrestore(&vdomain->mappings_lock, flags); + + return unmapped; +} + +/* + * viommu_replay_mappings - re-send MAP requests + * + * When reattaching a domain that was previously detached from all endpoints, + * mappings were deleted from the device. Re-create the mappings available in + * the internal tree. + */ +static int viommu_replay_mappings(struct viommu_domain *vdomain) +{ + int ret = 0; + unsigned long flags; + struct viommu_mapping *mapping; + struct interval_tree_node *node; + struct virtio_iommu_req_map map; + + spin_lock_irqsave(&vdomain->mappings_lock, flags); + node = interval_tree_iter_first(&vdomain->mappings, 0, -1UL); + while (node) { + mapping = container_of(node, struct viommu_mapping, iova); + map = (struct virtio_iommu_req_map) { + .head.type = VIRTIO_IOMMU_T_MAP, + .domain = cpu_to_le32(vdomain->id), + .virt_start = cpu_to_le64(mapping->iova.start), + .virt_end = cpu_to_le64(mapping->iova.last), + .phys_start = cpu_to_le64(mapping->paddr), + .flags = cpu_to_le32(mapping->flags), + }; + + ret = viommu_send_req_sync(vdomain->viommu, &map, sizeof(map)); + if (ret) + break; + + node = interval_tree_iter_next(node, 0, -1UL); + } + spin_unlock_irqrestore(&vdomain->mappings_lock, flags); + + return ret; +} + +static int viommu_add_resv_mem(struct viommu_endpoint *vdev, + struct virtio_iommu_probe_resv_mem *mem, + size_t len) +{ + size_t size; + u64 start64, end64; + phys_addr_t start, end; + struct iommu_resv_region *region = NULL; + unsigned long prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + + start = start64 = le64_to_cpu(mem->start); + end = end64 = le64_to_cpu(mem->end); + size = end64 - start64 + 1; + + /* Catch any overflow, including the unlikely end64 - start64 + 1 = 0 */ + if (start != start64 || end != end64 || size < end64 - start64) + return -EOVERFLOW; + + if (len < sizeof(*mem)) + return -EINVAL; + + switch (mem->subtype) { + default: + dev_warn(vdev->dev, "unknown resv mem subtype 0x%x\n", + mem->subtype); + /* Fall-through */ + case VIRTIO_IOMMU_RESV_MEM_T_RESERVED: + region = iommu_alloc_resv_region(start, size, 0, + IOMMU_RESV_RESERVED); + break; + case VIRTIO_IOMMU_RESV_MEM_T_MSI: + region = iommu_alloc_resv_region(start, size, prot, + IOMMU_RESV_MSI); + break; + } + if (!region) + return -ENOMEM; + + list_add(&vdev->resv_regions, ®ion->list); + return 0; +} + +static int viommu_probe_endpoint(struct viommu_dev *viommu, struct device *dev) +{ + int ret; + u16 type, len; + size_t cur = 0; + size_t probe_len; + struct virtio_iommu_req_probe *probe; + struct virtio_iommu_probe_property *prop; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct viommu_endpoint *vdev = fwspec->iommu_priv; + + if (!fwspec->num_ids) + return -EINVAL; + + probe_len = sizeof(*probe) + viommu->probe_size + + sizeof(struct virtio_iommu_req_tail); + probe = kzalloc(probe_len, GFP_KERNEL); + if (!probe) + return -ENOMEM; + + probe->head.type = VIRTIO_IOMMU_T_PROBE; + /* + * For now, assume that properties of an endpoint that outputs multiple + * IDs are consistent. Only probe the first one. + */ + probe->endpoint = cpu_to_le32(fwspec->ids[0]); + + ret = viommu_send_req_sync(viommu, probe, probe_len); + if (ret) + goto out_free; + + prop = (void *)probe->properties; + type = le16_to_cpu(prop->type) & VIRTIO_IOMMU_PROBE_T_MASK; + + while (type != VIRTIO_IOMMU_PROBE_T_NONE && + cur < viommu->probe_size) { + len = le16_to_cpu(prop->length) + sizeof(*prop); + + switch (type) { + case VIRTIO_IOMMU_PROBE_T_RESV_MEM: + ret = viommu_add_resv_mem(vdev, (void *)prop, len); + break; + default: + dev_err(dev, "unknown viommu prop 0x%x\n", type); + } + + if (ret) + dev_err(dev, "failed to parse viommu prop 0x%x\n", type); + + cur += len; + if (cur >= viommu->probe_size) + break; + + prop = (void *)probe->properties + cur; + type = le16_to_cpu(prop->type) & VIRTIO_IOMMU_PROBE_T_MASK; + } + +out_free: + kfree(probe); + return ret; +} + +static int viommu_fault_handler(struct viommu_dev *viommu, + struct virtio_iommu_fault *fault) +{ + char *reason_str; + + u8 reason = fault->reason; + u32 flags = le32_to_cpu(fault->flags); + u32 endpoint = le32_to_cpu(fault->endpoint); + u64 address = le64_to_cpu(fault->address); + + switch (reason) { + case VIRTIO_IOMMU_FAULT_R_DOMAIN: + reason_str = "domain"; + break; + case VIRTIO_IOMMU_FAULT_R_MAPPING: + reason_str = "page"; + break; + case VIRTIO_IOMMU_FAULT_R_UNKNOWN: + default: + reason_str = "unknown"; + break; + } + + /* TODO: find EP by ID and report_iommu_fault */ + if (flags & VIRTIO_IOMMU_FAULT_F_ADDRESS) + dev_err_ratelimited(viommu->dev, "%s fault from EP %u at %#llx [%s%s%s]\n", + reason_str, endpoint, address, + flags & VIRTIO_IOMMU_FAULT_F_READ ? "R" : "", + flags & VIRTIO_IOMMU_FAULT_F_WRITE ? "W" : "", + flags & VIRTIO_IOMMU_FAULT_F_EXEC ? "X" : ""); + else + dev_err_ratelimited(viommu->dev, "%s fault from EP %u\n", + reason_str, endpoint); + return 0; +} + +static void viommu_event_handler(struct virtqueue *vq) +{ + int ret; + unsigned int len; + struct scatterlist sg[1]; + struct viommu_event *evt; + struct viommu_dev *viommu = vq->vdev->priv; + + while ((evt = virtqueue_get_buf(vq, &len)) != NULL) { + if (len > sizeof(*evt)) { + dev_err(viommu->dev, + "invalid event buffer (len %u != %zu)\n", + len, sizeof(*evt)); + } else if (!(evt->head & VIOMMU_FAULT_RESV_MASK)) { + viommu_fault_handler(viommu, &evt->fault); + } + + sg_init_one(sg, evt, sizeof(*evt)); + ret = virtqueue_add_inbuf(vq, sg, 1, evt, GFP_ATOMIC); + if (ret) + dev_err(viommu->dev, "could not add event buffer\n"); + } + + virtqueue_kick(vq); +} + +/* IOMMU API */ + +static struct iommu_domain *viommu_domain_alloc(unsigned type) +{ + struct viommu_domain *vdomain; + + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) + return NULL; + + vdomain = kzalloc(sizeof(*vdomain), GFP_KERNEL); + if (!vdomain) + return NULL; + + mutex_init(&vdomain->mutex); + spin_lock_init(&vdomain->mappings_lock); + vdomain->mappings = RB_ROOT_CACHED; + + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&vdomain->domain)) { + kfree(vdomain); + return NULL; + } + + return &vdomain->domain; +} + +static int viommu_domain_finalise(struct viommu_dev *viommu, + struct iommu_domain *domain) +{ + int ret; + struct viommu_domain *vdomain = to_viommu_domain(domain); + unsigned int max_domain = viommu->domain_bits > 31 ? ~0 : + (1U << viommu->domain_bits) - 1; + + vdomain->viommu = viommu; + + domain->pgsize_bitmap = viommu->pgsize_bitmap; + domain->geometry = viommu->geometry; + + ret = ida_alloc_max(&viommu->domain_ids, max_domain, GFP_KERNEL); + if (ret >= 0) + vdomain->id = (unsigned int)ret; + + return ret > 0 ? 0 : ret; +} + +static void viommu_domain_free(struct iommu_domain *domain) +{ + struct viommu_domain *vdomain = to_viommu_domain(domain); + + iommu_put_dma_cookie(domain); + + /* Free all remaining mappings (size 2^64) */ + viommu_del_mappings(vdomain, 0, 0); + + if (vdomain->viommu) + ida_free(&vdomain->viommu->domain_ids, vdomain->id); + + kfree(vdomain); +} + +static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev) +{ + int i; + int ret = 0; + struct virtio_iommu_req_attach req; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct viommu_endpoint *vdev = fwspec->iommu_priv; + struct viommu_domain *vdomain = to_viommu_domain(domain); + + mutex_lock(&vdomain->mutex); + if (!vdomain->viommu) { + /* + * Properly initialize the domain now that we know which viommu + * owns it. + */ + ret = viommu_domain_finalise(vdev->viommu, domain); + } else if (vdomain->viommu != vdev->viommu) { + dev_err(dev, "cannot attach to foreign vIOMMU\n"); + ret = -EXDEV; + } + mutex_unlock(&vdomain->mutex); + + if (ret) + return ret; + + /* + * In the virtio-iommu device, when attaching the endpoint to a new + * domain, it is detached from the old one and, if as as a result the + * old domain isn't attached to any endpoint, all mappings are removed + * from the old domain and it is freed. + * + * In the driver the old domain still exists, and its mappings will be + * recreated if it gets reattached to an endpoint. Otherwise it will be + * freed explicitly. + * + * vdev->vdomain is protected by group->mutex + */ + if (vdev->vdomain) + vdev->vdomain->nr_endpoints--; + + req = (struct virtio_iommu_req_attach) { + .head.type = VIRTIO_IOMMU_T_ATTACH, + .domain = cpu_to_le32(vdomain->id), + }; + + for (i = 0; i < fwspec->num_ids; i++) { + req.endpoint = cpu_to_le32(fwspec->ids[i]); + + ret = viommu_send_req_sync(vdomain->viommu, &req, sizeof(req)); + if (ret) + return ret; + } + + if (!vdomain->nr_endpoints) { + /* + * This endpoint is the first to be attached to the domain. + * Replay existing mappings (e.g. SW MSI). + */ + ret = viommu_replay_mappings(vdomain); + if (ret) + return ret; + } + + vdomain->nr_endpoints++; + vdev->vdomain = vdomain; + + return 0; +} + +static int viommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + int ret; + int flags; + struct virtio_iommu_req_map map; + struct viommu_domain *vdomain = to_viommu_domain(domain); + + flags = (prot & IOMMU_READ ? VIRTIO_IOMMU_MAP_F_READ : 0) | + (prot & IOMMU_WRITE ? VIRTIO_IOMMU_MAP_F_WRITE : 0) | + (prot & IOMMU_MMIO ? VIRTIO_IOMMU_MAP_F_MMIO : 0); + + ret = viommu_add_mapping(vdomain, iova, paddr, size, flags); + if (ret) + return ret; + + map = (struct virtio_iommu_req_map) { + .head.type = VIRTIO_IOMMU_T_MAP, + .domain = cpu_to_le32(vdomain->id), + .virt_start = cpu_to_le64(iova), + .phys_start = cpu_to_le64(paddr), + .virt_end = cpu_to_le64(iova + size - 1), + .flags = cpu_to_le32(flags), + }; + + if (!vdomain->nr_endpoints) + return 0; + + ret = viommu_send_req_sync(vdomain->viommu, &map, sizeof(map)); + if (ret) + viommu_del_mappings(vdomain, iova, size); + + return ret; +} + +static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t size) +{ + int ret = 0; + size_t unmapped; + struct virtio_iommu_req_unmap unmap; + struct viommu_domain *vdomain = to_viommu_domain(domain); + + unmapped = viommu_del_mappings(vdomain, iova, size); + if (unmapped < size) + return 0; + + /* Device already removed all mappings after detach. */ + if (!vdomain->nr_endpoints) + return unmapped; + + unmap = (struct virtio_iommu_req_unmap) { + .head.type = VIRTIO_IOMMU_T_UNMAP, + .domain = cpu_to_le32(vdomain->id), + .virt_start = cpu_to_le64(iova), + .virt_end = cpu_to_le64(iova + unmapped - 1), + }; + + ret = viommu_add_req(vdomain->viommu, &unmap, sizeof(unmap)); + return ret ? 0 : unmapped; +} + +static phys_addr_t viommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + u64 paddr = 0; + unsigned long flags; + struct viommu_mapping *mapping; + struct interval_tree_node *node; + struct viommu_domain *vdomain = to_viommu_domain(domain); + + spin_lock_irqsave(&vdomain->mappings_lock, flags); + node = interval_tree_iter_first(&vdomain->mappings, iova, iova); + if (node) { + mapping = container_of(node, struct viommu_mapping, iova); + paddr = mapping->paddr + (iova - mapping->iova.start); + } + spin_unlock_irqrestore(&vdomain->mappings_lock, flags); + + return paddr; +} + +static void viommu_iotlb_sync(struct iommu_domain *domain) +{ + struct viommu_domain *vdomain = to_viommu_domain(domain); + + viommu_sync_req(vdomain->viommu); +} + +static void viommu_get_resv_regions(struct device *dev, struct list_head *head) +{ + struct iommu_resv_region *entry, *new_entry, *msi = NULL; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct viommu_endpoint *vdev = fwspec->iommu_priv; + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + + list_for_each_entry(entry, &vdev->resv_regions, list) { + if (entry->type == IOMMU_RESV_MSI) + msi = entry; + + new_entry = kmemdup(entry, sizeof(*entry), GFP_KERNEL); + if (!new_entry) + return; + list_add_tail(&new_entry->list, head); + } + + /* + * If the device didn't register any bypass MSI window, add a + * software-mapped region. + */ + if (!msi) { + msi = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, + prot, IOMMU_RESV_SW_MSI); + if (!msi) + return; + + list_add_tail(&msi->list, head); + } + + iommu_dma_get_resv_regions(dev, head); +} + +static void viommu_put_resv_regions(struct device *dev, struct list_head *head) +{ + struct iommu_resv_region *entry, *next; + + list_for_each_entry_safe(entry, next, head, list) + kfree(entry); +} + +static struct iommu_ops viommu_ops; +static struct virtio_driver virtio_iommu_drv; + +static int viommu_match_node(struct device *dev, const void *data) +{ + return dev->parent->fwnode == data; +} + +static struct viommu_dev *viommu_get_by_fwnode(struct fwnode_handle *fwnode) +{ + struct device *dev = driver_find_device(&virtio_iommu_drv.driver, NULL, + fwnode, viommu_match_node); + put_device(dev); + + return dev ? dev_to_virtio(dev)->priv : NULL; +} + +static int viommu_add_device(struct device *dev) +{ + int ret; + struct iommu_group *group; + struct viommu_endpoint *vdev; + struct viommu_dev *viommu = NULL; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + + if (!fwspec || fwspec->ops != &viommu_ops) + return -ENODEV; + + viommu = viommu_get_by_fwnode(fwspec->iommu_fwnode); + if (!viommu) + return -ENODEV; + + vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); + if (!vdev) + return -ENOMEM; + + vdev->dev = dev; + vdev->viommu = viommu; + INIT_LIST_HEAD(&vdev->resv_regions); + fwspec->iommu_priv = vdev; + + if (viommu->probe_size) { + /* Get additional information for this endpoint */ + ret = viommu_probe_endpoint(viommu, dev); + if (ret) + goto err_free_dev; + } + + ret = iommu_device_link(&viommu->iommu, dev); + if (ret) + goto err_free_dev; + + /* + * Last step creates a default domain and attaches to it. Everything + * must be ready. + */ + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) { + ret = PTR_ERR(group); + goto err_unlink_dev; + } + + iommu_group_put(group); + + return PTR_ERR_OR_ZERO(group); + +err_unlink_dev: + iommu_device_unlink(&viommu->iommu, dev); +err_free_dev: + viommu_put_resv_regions(dev, &vdev->resv_regions); + kfree(vdev); + + return ret; +} + +static void viommu_remove_device(struct device *dev) +{ + struct viommu_endpoint *vdev; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + + if (!fwspec || fwspec->ops != &viommu_ops) + return; + + vdev = fwspec->iommu_priv; + + iommu_group_remove_device(dev); + iommu_device_unlink(&vdev->viommu->iommu, dev); + viommu_put_resv_regions(dev, &vdev->resv_regions); + kfree(vdev); +} + +static struct iommu_group *viommu_device_group(struct device *dev) +{ + if (dev_is_pci(dev)) + return pci_device_group(dev); + else + return generic_device_group(dev); +} + +static int viommu_of_xlate(struct device *dev, struct of_phandle_args *args) +{ + return iommu_fwspec_add_ids(dev, args->args, 1); +} + +static struct iommu_ops viommu_ops = { + .domain_alloc = viommu_domain_alloc, + .domain_free = viommu_domain_free, + .attach_dev = viommu_attach_dev, + .map = viommu_map, + .unmap = viommu_unmap, + .iova_to_phys = viommu_iova_to_phys, + .iotlb_sync = viommu_iotlb_sync, + .add_device = viommu_add_device, + .remove_device = viommu_remove_device, + .device_group = viommu_device_group, + .get_resv_regions = viommu_get_resv_regions, + .put_resv_regions = viommu_put_resv_regions, + .of_xlate = viommu_of_xlate, +}; + +static int viommu_init_vqs(struct viommu_dev *viommu) +{ + struct virtio_device *vdev = dev_to_virtio(viommu->dev); + const char *names[] = { "request", "event" }; + vq_callback_t *callbacks[] = { + NULL, /* No async requests */ + viommu_event_handler, + }; + + return virtio_find_vqs(vdev, VIOMMU_NR_VQS, viommu->vqs, callbacks, + names, NULL); +} + +static int viommu_fill_evtq(struct viommu_dev *viommu) +{ + int i, ret; + struct scatterlist sg[1]; + struct viommu_event *evts; + struct virtqueue *vq = viommu->vqs[VIOMMU_EVENT_VQ]; + size_t nr_evts = vq->num_free; + + viommu->evts = evts = devm_kmalloc_array(viommu->dev, nr_evts, + sizeof(*evts), GFP_KERNEL); + if (!evts) + return -ENOMEM; + + for (i = 0; i < nr_evts; i++) { + sg_init_one(sg, &evts[i], sizeof(*evts)); + ret = virtqueue_add_inbuf(vq, sg, 1, &evts[i], GFP_KERNEL); + if (ret) + return ret; + } + + return 0; +} + +static int viommu_probe(struct virtio_device *vdev) +{ + struct device *parent_dev = vdev->dev.parent; + struct viommu_dev *viommu = NULL; + struct device *dev = &vdev->dev; + u64 input_start = 0; + u64 input_end = -1UL; + int ret; + + if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1) || + !virtio_has_feature(vdev, VIRTIO_IOMMU_F_MAP_UNMAP)) + return -ENODEV; + + viommu = devm_kzalloc(dev, sizeof(*viommu), GFP_KERNEL); + if (!viommu) + return -ENOMEM; + + spin_lock_init(&viommu->request_lock); + ida_init(&viommu->domain_ids); + viommu->dev = dev; + viommu->vdev = vdev; + INIT_LIST_HEAD(&viommu->requests); + + ret = viommu_init_vqs(viommu); + if (ret) + return ret; + + virtio_cread(vdev, struct virtio_iommu_config, page_size_mask, + &viommu->pgsize_bitmap); + + if (!viommu->pgsize_bitmap) { + ret = -EINVAL; + goto err_free_vqs; + } + + viommu->domain_bits = 32; + + /* Optional features */ + virtio_cread_feature(vdev, VIRTIO_IOMMU_F_INPUT_RANGE, + struct virtio_iommu_config, input_range.start, + &input_start); + + virtio_cread_feature(vdev, VIRTIO_IOMMU_F_INPUT_RANGE, + struct virtio_iommu_config, input_range.end, + &input_end); + + virtio_cread_feature(vdev, VIRTIO_IOMMU_F_DOMAIN_BITS, + struct virtio_iommu_config, domain_bits, + &viommu->domain_bits); + + virtio_cread_feature(vdev, VIRTIO_IOMMU_F_PROBE, + struct virtio_iommu_config, probe_size, + &viommu->probe_size); + + viommu->geometry = (struct iommu_domain_geometry) { + .aperture_start = input_start, + .aperture_end = input_end, + .force_aperture = true, + }; + + viommu_ops.pgsize_bitmap = viommu->pgsize_bitmap; + + virtio_device_ready(vdev); + + /* Populate the event queue with buffers */ + ret = viommu_fill_evtq(viommu); + if (ret) + goto err_free_vqs; + + ret = iommu_device_sysfs_add(&viommu->iommu, dev, NULL, "%s", + virtio_bus_name(vdev)); + if (ret) + goto err_free_vqs; + + iommu_device_set_ops(&viommu->iommu, &viommu_ops); + iommu_device_set_fwnode(&viommu->iommu, parent_dev->fwnode); + + iommu_device_register(&viommu->iommu); + +#ifdef CONFIG_PCI + if (pci_bus_type.iommu_ops != &viommu_ops) { + pci_request_acs(); + ret = bus_set_iommu(&pci_bus_type, &viommu_ops); + if (ret) + goto err_unregister; + } +#endif +#ifdef CONFIG_ARM_AMBA + if (amba_bustype.iommu_ops != &viommu_ops) { + ret = bus_set_iommu(&amba_bustype, &viommu_ops); + if (ret) + goto err_unregister; + } +#endif + if (platform_bus_type.iommu_ops != &viommu_ops) { + ret = bus_set_iommu(&platform_bus_type, &viommu_ops); + if (ret) + goto err_unregister; + } + + vdev->priv = viommu; + + dev_info(dev, "input address: %u bits\n", + order_base_2(viommu->geometry.aperture_end)); + dev_info(dev, "page mask: %#llx\n", viommu->pgsize_bitmap); + + return 0; + +err_unregister: + iommu_device_sysfs_remove(&viommu->iommu); + iommu_device_unregister(&viommu->iommu); +err_free_vqs: + vdev->config->del_vqs(vdev); + + return ret; +} + +static void viommu_remove(struct virtio_device *vdev) +{ + struct viommu_dev *viommu = vdev->priv; + + iommu_device_sysfs_remove(&viommu->iommu); + iommu_device_unregister(&viommu->iommu); + + /* Stop all virtqueues */ + vdev->config->reset(vdev); + vdev->config->del_vqs(vdev); + + dev_info(&vdev->dev, "device removed\n"); +} + +static void viommu_config_changed(struct virtio_device *vdev) +{ + dev_warn(&vdev->dev, "config changed\n"); +} + +static unsigned int features[] = { + VIRTIO_IOMMU_F_MAP_UNMAP, + VIRTIO_IOMMU_F_DOMAIN_BITS, + VIRTIO_IOMMU_F_INPUT_RANGE, + VIRTIO_IOMMU_F_PROBE, +}; + +static struct virtio_device_id id_table[] = { + { VIRTIO_ID_IOMMU, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static struct virtio_driver virtio_iommu_drv = { + .driver.name = KBUILD_MODNAME, + .driver.owner = THIS_MODULE, + .id_table = id_table, + .feature_table = features, + .feature_table_size = ARRAY_SIZE(features), + .probe = viommu_probe, + .remove = viommu_remove, + .config_changed = viommu_config_changed, +}; + +module_virtio_driver(virtio_iommu_drv); + +MODULE_DESCRIPTION("Virtio IOMMU driver"); +MODULE_AUTHOR("Jean-Philippe Brucker <jean-philippe.brucker@arm.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index ec8b27e20de3..caaee8032afe 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -881,7 +881,7 @@ void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type) EXPORT_SYMBOL_GPL(dm_table_set_type); /* validate the dax capability of the target device span */ -static int device_supports_dax(struct dm_target *ti, struct dm_dev *dev, +int device_supports_dax(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { int blocksize = *(int *) data; @@ -890,7 +890,15 @@ static int device_supports_dax(struct dm_target *ti, struct dm_dev *dev, start, len); } -bool dm_table_supports_dax(struct dm_table *t, int blocksize) +/* Check devices support synchronous DAX */ +static int device_synchronous(struct dm_target *ti, struct dm_dev *dev, + sector_t start, sector_t len, void *data) +{ + return dax_synchronous(dev->dax_dev); +} + +bool dm_table_supports_dax(struct dm_table *t, + iterate_devices_callout_fn iterate_fn, int *blocksize) { struct dm_target *ti; unsigned i; @@ -903,8 +911,7 @@ bool dm_table_supports_dax(struct dm_table *t, int blocksize) return false; if (!ti->type->iterate_devices || - !ti->type->iterate_devices(ti, device_supports_dax, - &blocksize)) + !ti->type->iterate_devices(ti, iterate_fn, blocksize)) return false; } @@ -940,6 +947,7 @@ static int dm_table_determine_type(struct dm_table *t) struct dm_target *tgt; struct list_head *devices = dm_table_get_devices(t); enum dm_queue_mode live_md_type = dm_get_md_type(t->md); + int page_size = PAGE_SIZE; if (t->type != DM_TYPE_NONE) { /* target already set the table's type */ @@ -984,7 +992,7 @@ static int dm_table_determine_type(struct dm_table *t) verify_bio_based: /* We must use this table as bio-based */ t->type = DM_TYPE_BIO_BASED; - if (dm_table_supports_dax(t, PAGE_SIZE) || + if (dm_table_supports_dax(t, device_supports_dax, &page_size) || (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) { t->type = DM_TYPE_DAX_BIO_BASED; } else { @@ -1883,6 +1891,7 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, struct queue_limits *limits) { bool wc = false, fua = false; + int page_size = PAGE_SIZE; /* * Copy table's limits to the DM device's request_queue @@ -1910,8 +1919,11 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, } blk_queue_write_cache(q, wc, fua); - if (dm_table_supports_dax(t, PAGE_SIZE)) + if (dm_table_supports_dax(t, device_supports_dax, &page_size)) { blk_queue_flag_set(QUEUE_FLAG_DAX, q); + if (dm_table_supports_dax(t, device_synchronous, NULL)) + set_dax_synchronous(t->md->dax_dev); + } else blk_queue_flag_clear(QUEUE_FLAG_DAX, q); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 61f1152b74e9..d0beef033e2f 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1117,7 +1117,7 @@ static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bd if (!map) return false; - ret = dm_table_supports_dax(map, blocksize); + ret = dm_table_supports_dax(map, device_supports_dax, &blocksize); dm_put_live_table(md, srcu_idx); @@ -1989,7 +1989,8 @@ static struct mapped_device *alloc_dev(int minor) sprintf(md->disk->disk_name, "dm-%d", minor); if (IS_ENABLED(CONFIG_DAX_DRIVER)) { - md->dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops); + md->dax_dev = alloc_dax(md, md->disk->disk_name, + &dm_dax_ops, 0); if (!md->dax_dev) goto bad; } diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 17e3db54404c..0475673337f3 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -72,7 +72,10 @@ bool dm_table_bio_based(struct dm_table *t); bool dm_table_request_based(struct dm_table *t); void dm_table_free_md_mempools(struct dm_table *t); struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t); -bool dm_table_supports_dax(struct dm_table *t, int blocksize); +bool dm_table_supports_dax(struct dm_table *t, iterate_devices_callout_fn fn, + int *blocksize); +int device_supports_dax(struct dm_target *ti, struct dm_dev *dev, + sector_t start, sector_t len, void *data); void dm_lock_md_type(struct mapped_device *md); void dm_unlock_md_type(struct mapped_device *md); diff --git a/drivers/memory/jz4780-nemc.c b/drivers/memory/jz4780-nemc.c index 2a3f7ef1c8c4..b232ed279fc3 100644 --- a/drivers/memory/jz4780-nemc.c +++ b/drivers/memory/jz4780-nemc.c @@ -61,7 +61,7 @@ struct jz4780_nemc { * * Return: The number of unique NEMC banks referred to by the specified NEMC * child device. Unique here means that a device that references the same bank - * multiple times in the its "reg" property will only count once. + * multiple times in its "reg" property will only count once. */ unsigned int jz4780_nemc_num_banks(struct device *dev) { diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index 6765f10837ac..6e208a060a58 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -793,7 +793,7 @@ static const struct pci_device_id pci_endpoint_test_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x) }, { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x) }, { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0) }, - { PCI_DEVICE(PCI_VENDOR_ID_SYNOPSYS, 0xedda) }, + { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) }, { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654), .driver_data = (kernel_ulong_t)&am654_data }, diff --git a/drivers/net/fddi/defza.c b/drivers/net/fddi/defza.c index c5cae8e74dc4..060712c666bf 100644 --- a/drivers/net/fddi/defza.c +++ b/drivers/net/fddi/defza.c @@ -33,6 +33,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/io-64-nonatomic-lo-hi.h> #include <linux/ioport.h> #include <linux/kernel.h> #include <linux/list.h> diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_usb_core.c b/drivers/net/wireless/mediatek/mt76/mt76x02_usb_core.c index 5e4f3a8c5784..e4332d5a5757 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_usb_core.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_usb_core.c @@ -53,7 +53,7 @@ int mt76x02u_skb_dma_info(struct sk_buff *skb, int port, u32 flags) pad = round_up(skb->len, 4) + 4 - skb->len; /* First packet of a A-MSDU burst keeps track of the whole burst - * length, need to update lenght of it and the last packet. + * length, need to update length of it and the last packet. */ skb_walk_frags(skb, iter) { last = iter; diff --git a/drivers/nvdimm/Makefile b/drivers/nvdimm/Makefile index 6f2a088afad6..cefe233e0b52 100644 --- a/drivers/nvdimm/Makefile +++ b/drivers/nvdimm/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_ND_BTT) += nd_btt.o obj-$(CONFIG_ND_BLK) += nd_blk.o obj-$(CONFIG_X86_PMEM_LEGACY) += nd_e820.o obj-$(CONFIG_OF_PMEM) += of_pmem.o +obj-$(CONFIG_VIRTIO_PMEM) += virtio_pmem.o nd_virtio.o nd_pmem-y := pmem.o diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c index 26c1c7618891..2985ca949912 100644 --- a/drivers/nvdimm/claim.c +++ b/drivers/nvdimm/claim.c @@ -255,7 +255,7 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns, struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); unsigned int sz_align = ALIGN(size + (offset & (512 - 1)), 512); sector_t sector = offset >> 9; - int rc = 0; + int rc = 0, ret = 0; if (unlikely(!size)) return 0; @@ -293,7 +293,9 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns, } memcpy_flushcache(nsio->addr + offset, buf, size); - nvdimm_flush(to_nd_region(ndns->dev.parent)); + ret = nvdimm_flush(to_nd_region(ndns->dev.parent), NULL); + if (ret) + rc = ret; return rc; } diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index a434a5964cb9..2d8d7e554877 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -1822,8 +1822,8 @@ static bool has_uuid_at_pos(struct nd_region *nd_region, u8 *uuid, && !guid_equal(&nd_set->type_guid, &nd_label->type_guid)) { dev_dbg(ndd->dev, "expect type_guid %pUb got %pUb\n", - nd_set->type_guid.b, - nd_label->type_guid.b); + &nd_set->type_guid, + &nd_label->type_guid); continue; } @@ -2227,8 +2227,8 @@ static struct device *create_namespace_blk(struct nd_region *nd_region, if (namespace_label_has(ndd, type_guid)) { if (!guid_equal(&nd_set->type_guid, &nd_label->type_guid)) { dev_dbg(ndd->dev, "expect type_guid %pUb got %pUb\n", - nd_set->type_guid.b, - nd_label->type_guid.b); + &nd_set->type_guid, + &nd_label->type_guid); return ERR_PTR(-EAGAIN); } diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index d24304c0e6d7..1b9955651379 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -155,6 +155,7 @@ struct nd_region { struct badblocks bb; struct nd_interleave_set *nd_set; struct nd_percpu_lane __percpu *lane; + int (*flush)(struct nd_region *nd_region, struct bio *bio); struct nd_mapping mapping[0]; }; diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c new file mode 100644 index 000000000000..10351d5b49fa --- /dev/null +++ b/drivers/nvdimm/nd_virtio.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * virtio_pmem.c: Virtio pmem Driver + * + * Discovers persistent memory range information + * from host and provides a virtio based flushing + * interface. + */ +#include "virtio_pmem.h" +#include "nd.h" + + /* The interrupt handler */ +void virtio_pmem_host_ack(struct virtqueue *vq) +{ + struct virtio_pmem *vpmem = vq->vdev->priv; + struct virtio_pmem_request *req_data, *req_buf; + unsigned long flags; + unsigned int len; + + spin_lock_irqsave(&vpmem->pmem_lock, flags); + while ((req_data = virtqueue_get_buf(vq, &len)) != NULL) { + req_data->done = true; + wake_up(&req_data->host_acked); + + if (!list_empty(&vpmem->req_list)) { + req_buf = list_first_entry(&vpmem->req_list, + struct virtio_pmem_request, list); + req_buf->wq_buf_avail = true; + wake_up(&req_buf->wq_buf); + list_del(&req_buf->list); + } + } + spin_unlock_irqrestore(&vpmem->pmem_lock, flags); +} +EXPORT_SYMBOL_GPL(virtio_pmem_host_ack); + + /* The request submission function */ +static int virtio_pmem_flush(struct nd_region *nd_region) +{ + struct virtio_device *vdev = nd_region->provider_data; + struct virtio_pmem *vpmem = vdev->priv; + struct virtio_pmem_request *req_data; + struct scatterlist *sgs[2], sg, ret; + unsigned long flags; + int err, err1; + + might_sleep(); + req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); + if (!req_data) + return -ENOMEM; + + req_data->done = false; + init_waitqueue_head(&req_data->host_acked); + init_waitqueue_head(&req_data->wq_buf); + INIT_LIST_HEAD(&req_data->list); + req_data->req.type = cpu_to_le32(VIRTIO_PMEM_REQ_TYPE_FLUSH); + sg_init_one(&sg, &req_data->req, sizeof(req_data->req)); + sgs[0] = &sg; + sg_init_one(&ret, &req_data->resp.ret, sizeof(req_data->resp)); + sgs[1] = &ret; + + spin_lock_irqsave(&vpmem->pmem_lock, flags); + /* + * If virtqueue_add_sgs returns -ENOSPC then req_vq virtual + * queue does not have free descriptor. We add the request + * to req_list and wait for host_ack to wake us up when free + * slots are available. + */ + while ((err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, + GFP_ATOMIC)) == -ENOSPC) { + + dev_info(&vdev->dev, "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); + req_data->wq_buf_avail = false; + list_add_tail(&req_data->list, &vpmem->req_list); + spin_unlock_irqrestore(&vpmem->pmem_lock, flags); + + /* A host response results in "host_ack" getting called */ + wait_event(req_data->wq_buf, req_data->wq_buf_avail); + spin_lock_irqsave(&vpmem->pmem_lock, flags); + } + err1 = virtqueue_kick(vpmem->req_vq); + spin_unlock_irqrestore(&vpmem->pmem_lock, flags); + /* + * virtqueue_add_sgs failed with error different than -ENOSPC, we can't + * do anything about that. + */ + if (err || !err1) { + dev_info(&vdev->dev, "failed to send command to virtio pmem device\n"); + err = -EIO; + } else { + /* A host repsonse results in "host_ack" getting called */ + wait_event(req_data->host_acked, req_data->done); + err = le32_to_cpu(req_data->resp.ret); + } + + kfree(req_data); + return err; +}; + +/* The asynchronous flush callback function */ +int async_pmem_flush(struct nd_region *nd_region, struct bio *bio) +{ + /* + * Create child bio for asynchronous flush and chain with + * parent bio. Otherwise directly call nd_region flush. + */ + if (bio && bio->bi_iter.bi_sector != -1) { + struct bio *child = bio_alloc(GFP_ATOMIC, 0); + + if (!child) + return -ENOMEM; + bio_copy_dev(child, bio); + child->bi_opf = REQ_PREFLUSH; + child->bi_iter.bi_sector = -1; + bio_chain(child, bio); + submit_bio(child); + return 0; + } + if (virtio_pmem_flush(nd_region)) + return -EIO; + + return 0; +}; +EXPORT_SYMBOL_GPL(async_pmem_flush); +MODULE_LICENSE("GPL"); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index e7d8cc9f41e8..2bf3acd69613 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -184,6 +184,7 @@ static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page, static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) { + int ret = 0; blk_status_t rc = 0; bool do_acct; unsigned long start; @@ -193,7 +194,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) struct nd_region *nd_region = to_region(pmem); if (bio->bi_opf & REQ_PREFLUSH) - nvdimm_flush(nd_region); + ret = nvdimm_flush(nd_region, bio); do_acct = nd_iostat_start(bio, &start); bio_for_each_segment(bvec, bio, iter) { @@ -208,7 +209,10 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) nd_iostat_end(bio, start); if (bio->bi_opf & REQ_FUA) - nvdimm_flush(nd_region); + ret = nvdimm_flush(nd_region, bio); + + if (ret) + bio->bi_status = errno_to_blk_status(ret); bio_endio(bio); return BLK_QC_T_NONE; @@ -362,6 +366,7 @@ static int pmem_attach_disk(struct device *dev, struct gendisk *disk; void *addr; int rc; + unsigned long flags = 0UL; pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL); if (!pmem) @@ -457,14 +462,15 @@ static int pmem_attach_disk(struct device *dev, nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_res); disk->bb = &pmem->bb; - dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops); + if (is_nvdimm_sync(nd_region)) + flags = DAXDEV_F_SYNC; + dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops, flags); if (!dax_dev) { put_disk(disk); return -ENOMEM; } dax_write_cache(dax_dev, nvdimm_has_cache(nd_region)); pmem->dax_dev = dax_dev; - gendev = disk_to_dev(disk); gendev->groups = pmem_attribute_groups; @@ -522,14 +528,14 @@ static int nd_pmem_remove(struct device *dev) sysfs_put(pmem->bb_state); pmem->bb_state = NULL; } - nvdimm_flush(to_nd_region(dev->parent)); + nvdimm_flush(to_nd_region(dev->parent), NULL); return 0; } static void nd_pmem_shutdown(struct device *dev) { - nvdimm_flush(to_nd_region(dev->parent)); + nvdimm_flush(to_nd_region(dev->parent), NULL); } static void nd_pmem_notify(struct device *dev, enum nvdimm_event event) diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 4fed9ce9c2fe..56f2227f192a 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -287,7 +287,9 @@ static ssize_t deep_flush_store(struct device *dev, struct device_attribute *att return rc; if (!flush) return -EINVAL; - nvdimm_flush(nd_region); + rc = nvdimm_flush(nd_region, NULL); + if (rc) + return rc; return len; } @@ -1077,6 +1079,11 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, dev->of_node = ndr_desc->of_node; nd_region->ndr_size = resource_size(ndr_desc->res); nd_region->ndr_start = ndr_desc->res->start; + if (ndr_desc->flush) + nd_region->flush = ndr_desc->flush; + else + nd_region->flush = NULL; + nd_device_register(dev); return nd_region; @@ -1117,11 +1124,24 @@ struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus, } EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create); +int nvdimm_flush(struct nd_region *nd_region, struct bio *bio) +{ + int rc = 0; + + if (!nd_region->flush) + rc = generic_nvdimm_flush(nd_region); + else { + if (nd_region->flush(nd_region, bio)) + rc = -EIO; + } + + return rc; +} /** * nvdimm_flush - flush any posted write queues between the cpu and pmem media * @nd_region: blk or interleaved pmem region */ -void nvdimm_flush(struct nd_region *nd_region) +int generic_nvdimm_flush(struct nd_region *nd_region) { struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev); int i, idx; @@ -1145,6 +1165,8 @@ void nvdimm_flush(struct nd_region *nd_region) if (ndrd_get_flush_wpq(ndrd, i, 0)) writeq(1, ndrd_get_flush_wpq(ndrd, i, idx)); wmb(); + + return 0; } EXPORT_SYMBOL_GPL(nvdimm_flush); @@ -1189,6 +1211,13 @@ int nvdimm_has_cache(struct nd_region *nd_region) } EXPORT_SYMBOL_GPL(nvdimm_has_cache); +bool is_nvdimm_sync(struct nd_region *nd_region) +{ + return is_nd_pmem(&nd_region->dev) && + !test_bit(ND_REGION_ASYNC, &nd_region->flags); +} +EXPORT_SYMBOL_GPL(is_nvdimm_sync); + struct conflict_context { struct nd_region *nd_region; resource_size_t start, size; diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c new file mode 100644 index 000000000000..5e3d07b47e0c --- /dev/null +++ b/drivers/nvdimm/virtio_pmem.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * virtio_pmem.c: Virtio pmem Driver + * + * Discovers persistent memory range information + * from host and registers the virtual pmem device + * with libnvdimm core. + */ +#include "virtio_pmem.h" +#include "nd.h" + +static struct virtio_device_id id_table[] = { + { VIRTIO_ID_PMEM, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + + /* Initialize virt queue */ +static int init_vq(struct virtio_pmem *vpmem) +{ + /* single vq */ + vpmem->req_vq = virtio_find_single_vq(vpmem->vdev, + virtio_pmem_host_ack, "flush_queue"); + if (IS_ERR(vpmem->req_vq)) + return PTR_ERR(vpmem->req_vq); + + spin_lock_init(&vpmem->pmem_lock); + INIT_LIST_HEAD(&vpmem->req_list); + + return 0; +}; + +static int virtio_pmem_probe(struct virtio_device *vdev) +{ + struct nd_region_desc ndr_desc = {}; + int nid = dev_to_node(&vdev->dev); + struct nd_region *nd_region; + struct virtio_pmem *vpmem; + struct resource res; + int err = 0; + + if (!vdev->config->get) { + dev_err(&vdev->dev, "%s failure: config access disabled\n", + __func__); + return -EINVAL; + } + + vpmem = devm_kzalloc(&vdev->dev, sizeof(*vpmem), GFP_KERNEL); + if (!vpmem) { + err = -ENOMEM; + goto out_err; + } + + vpmem->vdev = vdev; + vdev->priv = vpmem; + err = init_vq(vpmem); + if (err) { + dev_err(&vdev->dev, "failed to initialize virtio pmem vq's\n"); + goto out_err; + } + + virtio_cread(vpmem->vdev, struct virtio_pmem_config, + start, &vpmem->start); + virtio_cread(vpmem->vdev, struct virtio_pmem_config, + size, &vpmem->size); + + res.start = vpmem->start; + res.end = vpmem->start + vpmem->size - 1; + vpmem->nd_desc.provider_name = "virtio-pmem"; + vpmem->nd_desc.module = THIS_MODULE; + + vpmem->nvdimm_bus = nvdimm_bus_register(&vdev->dev, + &vpmem->nd_desc); + if (!vpmem->nvdimm_bus) { + dev_err(&vdev->dev, "failed to register device with nvdimm_bus\n"); + err = -ENXIO; + goto out_vq; + } + + dev_set_drvdata(&vdev->dev, vpmem->nvdimm_bus); + + ndr_desc.res = &res; + ndr_desc.numa_node = nid; + ndr_desc.flush = async_pmem_flush; + set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags); + set_bit(ND_REGION_ASYNC, &ndr_desc.flags); + nd_region = nvdimm_pmem_region_create(vpmem->nvdimm_bus, &ndr_desc); + if (!nd_region) { + dev_err(&vdev->dev, "failed to create nvdimm region\n"); + err = -ENXIO; + goto out_nd; + } + nd_region->provider_data = dev_to_virtio(nd_region->dev.parent->parent); + return 0; +out_nd: + nvdimm_bus_unregister(vpmem->nvdimm_bus); +out_vq: + vdev->config->del_vqs(vdev); +out_err: + return err; +} + +static void virtio_pmem_remove(struct virtio_device *vdev) +{ + struct nvdimm_bus *nvdimm_bus = dev_get_drvdata(&vdev->dev); + + nvdimm_bus_unregister(nvdimm_bus); + vdev->config->del_vqs(vdev); + vdev->config->reset(vdev); +} + +static struct virtio_driver virtio_pmem_driver = { + .driver.name = KBUILD_MODNAME, + .driver.owner = THIS_MODULE, + .id_table = id_table, + .probe = virtio_pmem_probe, + .remove = virtio_pmem_remove, +}; + +module_virtio_driver(virtio_pmem_driver); +MODULE_DEVICE_TABLE(virtio, id_table); +MODULE_DESCRIPTION("Virtio pmem driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h new file mode 100644 index 000000000000..0dddefe594c4 --- /dev/null +++ b/drivers/nvdimm/virtio_pmem.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * virtio_pmem.h: virtio pmem Driver + * + * Discovers persistent memory range information + * from host and provides a virtio based flushing + * interface. + **/ + +#ifndef _LINUX_VIRTIO_PMEM_H +#define _LINUX_VIRTIO_PMEM_H + +#include <linux/module.h> +#include <uapi/linux/virtio_pmem.h> +#include <linux/libnvdimm.h> +#include <linux/spinlock.h> + +struct virtio_pmem_request { + struct virtio_pmem_req req; + struct virtio_pmem_resp resp; + + /* Wait queue to process deferred work after ack from host */ + wait_queue_head_t host_acked; + bool done; + + /* Wait queue to process deferred work after virt queue buffer avail */ + wait_queue_head_t wq_buf; + bool wq_buf_avail; + struct list_head list; +}; + +struct virtio_pmem { + struct virtio_device *vdev; + + /* Virtio pmem request queue */ + struct virtqueue *req_vq; + + /* nvdimm bus registers virtio pmem device */ + struct nvdimm_bus *nvdimm_bus; + struct nvdimm_bus_descriptor nd_desc; + + /* List to store deferred work if virtqueue is full */ + struct list_head req_list; + + /* Synchronize virtqueue data */ + spinlock_t pmem_lock; + + /* Memory region information */ + __u64 start; + __u64 size; +}; + +void virtio_pmem_host_ack(struct virtqueue *vq); +int async_pmem_flush(struct nd_region *nd_region, struct bio *bio); +#endif diff --git a/drivers/of/base.c b/drivers/of/base.c index 20e0e7ee4edf..55e7f5bb0549 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -2294,8 +2294,12 @@ int of_map_rid(struct device_node *np, u32 rid, return 0; } - pr_err("%pOF: Invalid %s translation - no match for rid 0x%x on %pOF\n", - np, map_name, rid, target && *target ? *target : NULL); - return -EFAULT; + pr_info("%pOF: no %s translation for rid 0x%x on %pOF\n", np, map_name, + rid, target && *target ? *target : NULL); + + /* Bypasses translation */ + if (id_out) + *id_out = rid; + return 0; } EXPORT_SYMBOL_GPL(of_map_rid); diff --git a/drivers/pci/of.c b/drivers/pci/of.c index 73d5adec0a28..bc7b27a28795 100644 --- a/drivers/pci/of.c +++ b/drivers/pci/of.c @@ -22,12 +22,15 @@ void pci_set_of_node(struct pci_dev *dev) return; dev->dev.of_node = of_pci_find_child_device(dev->bus->dev.of_node, dev->devfn); + if (dev->dev.of_node) + dev->dev.fwnode = &dev->dev.of_node->fwnode; } void pci_release_of_node(struct pci_dev *dev) { of_node_put(dev->dev.of_node); dev->dev.of_node = NULL; + dev->dev.fwnode = NULL; } void pci_set_bus_of_node(struct pci_bus *bus) @@ -41,13 +44,18 @@ void pci_set_bus_of_node(struct pci_bus *bus) if (node && of_property_read_bool(node, "external-facing")) bus->self->untrusted = true; } + bus->dev.of_node = node; + + if (bus->dev.of_node) + bus->dev.fwnode = &bus->dev.of_node->fwnode; } void pci_release_bus_of_node(struct pci_bus *bus) { of_node_put(bus->dev.of_node); bus->dev.of_node = NULL; + bus->dev.fwnode = NULL; } struct device_node * __weak pcibios_get_phb_of_node(struct pci_bus *bus) diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index 18f3a8bad52f..ca28d27dae63 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -68,12 +68,12 @@ MODULE_LICENSE("GPL"); #define ASUS_FAN_CTRL_MANUAL 1 #define ASUS_FAN_CTRL_AUTO 2 -#define ASUS_FAN_MODE_NORMAL 0 -#define ASUS_FAN_MODE_OVERBOOST 1 -#define ASUS_FAN_MODE_OVERBOOST_MASK 0x01 -#define ASUS_FAN_MODE_SILENT 2 -#define ASUS_FAN_MODE_SILENT_MASK 0x02 -#define ASUS_FAN_MODES_MASK 0x03 +#define ASUS_FAN_BOOST_MODE_NORMAL 0 +#define ASUS_FAN_BOOST_MODE_OVERBOOST 1 +#define ASUS_FAN_BOOST_MODE_OVERBOOST_MASK 0x01 +#define ASUS_FAN_BOOST_MODE_SILENT 2 +#define ASUS_FAN_BOOST_MODE_SILENT_MASK 0x02 +#define ASUS_FAN_BOOST_MODES_MASK 0x03 #define USB_INTEL_XUSB2PR 0xD0 #define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI 0x9c31 @@ -182,9 +182,9 @@ struct asus_wmi { int asus_hwmon_num_fans; int asus_hwmon_pwm; - bool fan_mode_available; - u8 fan_mode_mask; - u8 fan_mode; + bool fan_boost_mode_available; + u8 fan_boost_mode_mask; + u8 fan_boost_mode; struct hotplug_slot hotplug_slot; struct mutex hotplug_lock; @@ -1487,14 +1487,15 @@ static int asus_wmi_fan_init(struct asus_wmi *asus) /* Fan mode *******************************************************************/ -static int fan_mode_check_present(struct asus_wmi *asus) +static int fan_boost_mode_check_present(struct asus_wmi *asus) { u32 result; int err; - asus->fan_mode_available = false; + asus->fan_boost_mode_available = false; - err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_FAN_MODE, &result); + err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_FAN_BOOST_MODE, + &result); if (err) { if (err == -ENODEV) return 0; @@ -1503,72 +1504,77 @@ static int fan_mode_check_present(struct asus_wmi *asus) } if ((result & ASUS_WMI_DSTS_PRESENCE_BIT) && - (result & ASUS_FAN_MODES_MASK)) { - asus->fan_mode_available = true; - asus->fan_mode_mask = result & ASUS_FAN_MODES_MASK; + (result & ASUS_FAN_BOOST_MODES_MASK)) { + asus->fan_boost_mode_available = true; + asus->fan_boost_mode_mask = result & ASUS_FAN_BOOST_MODES_MASK; } return 0; } -static int fan_mode_write(struct asus_wmi *asus) +static int fan_boost_mode_write(struct asus_wmi *asus) { int err; u8 value; u32 retval; - value = asus->fan_mode; + value = asus->fan_boost_mode; - pr_info("Set fan mode: %u\n", value); - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_FAN_MODE, value, &retval); + pr_info("Set fan boost mode: %u\n", value); + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_FAN_BOOST_MODE, value, + &retval); if (err) { - pr_warn("Failed to set fan mode: %d\n", err); + pr_warn("Failed to set fan boost mode: %d\n", err); return err; } if (retval != 1) { - pr_warn("Failed to set fan mode (retval): 0x%x\n", retval); + pr_warn("Failed to set fan boost mode (retval): 0x%x\n", + retval); return -EIO; } return 0; } -static int fan_mode_switch_next(struct asus_wmi *asus) +static int fan_boost_mode_switch_next(struct asus_wmi *asus) { - if (asus->fan_mode == ASUS_FAN_MODE_NORMAL) { - if (asus->fan_mode_mask & ASUS_FAN_MODE_OVERBOOST_MASK) - asus->fan_mode = ASUS_FAN_MODE_OVERBOOST; - else if (asus->fan_mode_mask & ASUS_FAN_MODE_SILENT_MASK) - asus->fan_mode = ASUS_FAN_MODE_SILENT; - } else if (asus->fan_mode == ASUS_FAN_MODE_OVERBOOST) { - if (asus->fan_mode_mask & ASUS_FAN_MODE_SILENT_MASK) - asus->fan_mode = ASUS_FAN_MODE_SILENT; + u8 mask = asus->fan_boost_mode_mask; + + if (asus->fan_boost_mode == ASUS_FAN_BOOST_MODE_NORMAL) { + if (mask & ASUS_FAN_BOOST_MODE_OVERBOOST_MASK) + asus->fan_boost_mode = ASUS_FAN_BOOST_MODE_OVERBOOST; + else if (mask & ASUS_FAN_BOOST_MODE_SILENT_MASK) + asus->fan_boost_mode = ASUS_FAN_BOOST_MODE_SILENT; + } else if (asus->fan_boost_mode == ASUS_FAN_BOOST_MODE_OVERBOOST) { + if (mask & ASUS_FAN_BOOST_MODE_SILENT_MASK) + asus->fan_boost_mode = ASUS_FAN_BOOST_MODE_SILENT; else - asus->fan_mode = ASUS_FAN_MODE_NORMAL; + asus->fan_boost_mode = ASUS_FAN_BOOST_MODE_NORMAL; } else { - asus->fan_mode = ASUS_FAN_MODE_NORMAL; + asus->fan_boost_mode = ASUS_FAN_BOOST_MODE_NORMAL; } - return fan_mode_write(asus); + return fan_boost_mode_write(asus); } -static ssize_t fan_mode_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t fan_boost_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct asus_wmi *asus = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", asus->fan_mode); + return scnprintf(buf, PAGE_SIZE, "%d\n", asus->fan_boost_mode); } -static ssize_t fan_mode_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t fan_boost_mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { int result; u8 new_mode; - struct asus_wmi *asus = dev_get_drvdata(dev); + u8 mask = asus->fan_boost_mode_mask; result = kstrtou8(buf, 10, &new_mode); if (result < 0) { @@ -1576,24 +1582,24 @@ static ssize_t fan_mode_store(struct device *dev, struct device_attribute *attr, return result; } - if (new_mode == ASUS_FAN_MODE_OVERBOOST) { - if (!(asus->fan_mode_mask & ASUS_FAN_MODE_OVERBOOST_MASK)) + if (new_mode == ASUS_FAN_BOOST_MODE_OVERBOOST) { + if (!(mask & ASUS_FAN_BOOST_MODE_OVERBOOST_MASK)) return -EINVAL; - } else if (new_mode == ASUS_FAN_MODE_SILENT) { - if (!(asus->fan_mode_mask & ASUS_FAN_MODE_SILENT_MASK)) + } else if (new_mode == ASUS_FAN_BOOST_MODE_SILENT) { + if (!(mask & ASUS_FAN_BOOST_MODE_SILENT_MASK)) return -EINVAL; - } else if (new_mode != ASUS_FAN_MODE_NORMAL) { + } else if (new_mode != ASUS_FAN_BOOST_MODE_NORMAL) { return -EINVAL; } - asus->fan_mode = new_mode; - fan_mode_write(asus); + asus->fan_boost_mode = new_mode; + fan_boost_mode_write(asus); return result; } -// Fan mode: 0 - normal, 1 - overboost, 2 - silent -static DEVICE_ATTR_RW(fan_mode); +// Fan boost mode: 0 - normal, 1 - overboost, 2 - silent +static DEVICE_ATTR_RW(fan_boost_mode); /* Backlight ******************************************************************/ @@ -1873,8 +1879,8 @@ static void asus_wmi_handle_event_code(int code, struct asus_wmi *asus) return; } - if (asus->fan_mode_available && code == NOTIFY_KBD_FBM) { - fan_mode_switch_next(asus); + if (asus->fan_boost_mode_available && code == NOTIFY_KBD_FBM) { + fan_boost_mode_switch_next(asus); return; } @@ -2034,7 +2040,7 @@ static struct attribute *platform_attributes[] = { &dev_attr_touchpad.attr, &dev_attr_lid_resume.attr, &dev_attr_als_enable.attr, - &dev_attr_fan_mode.attr, + &dev_attr_fan_boost_mode.attr, NULL }; @@ -2056,8 +2062,8 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj, devid = ASUS_WMI_DEVID_LID_RESUME; else if (attr == &dev_attr_als_enable.attr) devid = ASUS_WMI_DEVID_ALS_ENABLE; - else if (attr == &dev_attr_fan_mode.attr) - ok = asus->fan_mode_available; + else if (attr == &dev_attr_fan_boost_mode.attr) + ok = asus->fan_boost_mode_available; if (devid != -1) ok = !(asus_wmi_get_devstate_simple(asus, devid) < 0); @@ -2315,9 +2321,9 @@ static int asus_wmi_add(struct platform_device *pdev) if (err) goto fail_platform; - err = fan_mode_check_present(asus); + err = fan_boost_mode_check_present(asus); if (err) - goto fail_fan_mode; + goto fail_fan_boost_mode; err = asus_wmi_sysfs_init(asus->platform_device); if (err) @@ -2402,7 +2408,7 @@ fail_hwmon: fail_input: asus_wmi_sysfs_exit(asus->platform_device); fail_sysfs: -fail_fan_mode: +fail_fan_boost_mode: fail_platform: kfree(asus); return err; diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index 42d3798c88f0..dc1c1381d7fa 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -16,14 +16,17 @@ menuconfig POWERCAP if POWERCAP # Client driver configurations go here. +config INTEL_RAPL_CORE + tristate + config INTEL_RAPL - tristate "Intel RAPL Support" + tristate "Intel RAPL Support via MSR Interface" depends on X86 && IOSF_MBI - default n + select INTEL_RAPL_CORE ---help--- This enables support for the Intel Running Average Power Limit (RAPL) - technology which allows power limits to be enforced and monitored on - modern Intel processors (Sandy Bridge and later). + technology via MSR interface, which allows power limits to be enforced + and monitored on modern Intel processors (Sandy Bridge and later). In RAPL, the platform level settings are divided into domains for fine grained control. These domains include processor package, DRAM diff --git a/drivers/powercap/Makefile b/drivers/powercap/Makefile index 81c8ccaba6e7..7255c94ec61c 100644 --- a/drivers/powercap/Makefile +++ b/drivers/powercap/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_POWERCAP) += powercap_sys.o -obj-$(CONFIG_INTEL_RAPL) += intel_rapl.o +obj-$(CONFIG_INTEL_RAPL_CORE) += intel_rapl_common.o +obj-$(CONFIG_INTEL_RAPL) += intel_rapl_msr.o obj-$(CONFIG_IDLE_INJECT) += idle_inject.o diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl_common.c index 8692f6b79f93..9fd6dd342169 100644 --- a/drivers/powercap/intel_rapl.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Intel Running Average Power Limit (RAPL) Driver - * Copyright (c) 2013, Intel Corporation. + * Common code for Intel Running Average Power Limit (RAPL) support. + * Copyright (c) 2019, Intel Corporation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -18,9 +18,11 @@ #include <linux/cpu.h> #include <linux/powercap.h> #include <linux/suspend.h> -#include <asm/iosf_mbi.h> +#include <linux/intel_rapl.h> +#include <linux/processor.h> +#include <linux/platform_device.h> -#include <asm/processor.h> +#include <asm/iosf_mbi.h> #include <asm/cpu_device_id.h> #include <asm/intel-family.h> @@ -37,8 +39,8 @@ #define POWER_LIMIT2_MASK (0x7FFFULL<<32) #define POWER_LIMIT2_ENABLE BIT_ULL(47) #define POWER_LIMIT2_CLAMP BIT_ULL(48) -#define POWER_PACKAGE_LOCK BIT_ULL(63) -#define POWER_PP_LOCK BIT(31) +#define POWER_HIGH_LOCK BIT_ULL(63) +#define POWER_LOW_LOCK BIT(31) #define TIME_WINDOW1_MASK (0x7FULL<<17) #define TIME_WINDOW2_MASK (0x7FULL<<49) @@ -61,143 +63,38 @@ #define PP_POLICY_MASK 0x1F /* Non HW constants */ -#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */ +#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */ #define RAPL_PRIMITIVE_DUMMY BIT(2) #define TIME_WINDOW_MAX_MSEC 40000 #define TIME_WINDOW_MIN_MSEC 250 -#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */ +#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */ enum unit_type { - ARBITRARY_UNIT, /* no translation */ + ARBITRARY_UNIT, /* no translation */ POWER_UNIT, ENERGY_UNIT, TIME_UNIT, }; -enum rapl_domain_type { - RAPL_DOMAIN_PACKAGE, /* entire package/socket */ - RAPL_DOMAIN_PP0, /* core power plane */ - RAPL_DOMAIN_PP1, /* graphics uncore */ - RAPL_DOMAIN_DRAM,/* DRAM control_type */ - RAPL_DOMAIN_PLATFORM, /* PSys control_type */ - RAPL_DOMAIN_MAX, -}; - -enum rapl_domain_msr_id { - RAPL_DOMAIN_MSR_LIMIT, - RAPL_DOMAIN_MSR_STATUS, - RAPL_DOMAIN_MSR_PERF, - RAPL_DOMAIN_MSR_POLICY, - RAPL_DOMAIN_MSR_INFO, - RAPL_DOMAIN_MSR_MAX, -}; - /* per domain data, some are optional */ -enum rapl_primitives { - ENERGY_COUNTER, - POWER_LIMIT1, - POWER_LIMIT2, - FW_LOCK, - - PL1_ENABLE, /* power limit 1, aka long term */ - PL1_CLAMP, /* allow frequency to go below OS request */ - PL2_ENABLE, /* power limit 2, aka short term, instantaneous */ - PL2_CLAMP, - - TIME_WINDOW1, /* long term */ - TIME_WINDOW2, /* short term */ - THERMAL_SPEC_POWER, - MAX_POWER, - - MIN_POWER, - MAX_TIME_WINDOW, - THROTTLED_TIME, - PRIORITY_LEVEL, - - /* below are not raw primitive data */ - AVERAGE_POWER, - NR_RAPL_PRIMITIVES, -}; - #define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2) -/* Can be expanded to include events, etc.*/ -struct rapl_domain_data { - u64 primitives[NR_RAPL_PRIMITIVES]; - unsigned long timestamp; -}; - -struct msrl_action { - u32 msr_no; - u64 clear_mask; - u64 set_mask; - int err; -}; - #define DOMAIN_STATE_INACTIVE BIT(0) #define DOMAIN_STATE_POWER_LIMIT_SET BIT(1) #define DOMAIN_STATE_BIOS_LOCKED BIT(2) -#define NR_POWER_LIMITS (2) -struct rapl_power_limit { - struct powercap_zone_constraint *constraint; - int prim_id; /* primitive ID used to enable */ - struct rapl_domain *domain; - const char *name; - u64 last_power_limit; -}; - static const char pl1_name[] = "long_term"; static const char pl2_name[] = "short_term"; -struct rapl_package; -struct rapl_domain { - const char *name; - enum rapl_domain_type id; - int msrs[RAPL_DOMAIN_MSR_MAX]; - struct powercap_zone power_zone; - struct rapl_domain_data rdd; - struct rapl_power_limit rpl[NR_POWER_LIMITS]; - u64 attr_map; /* track capabilities */ - unsigned int state; - unsigned int domain_energy_unit; - struct rapl_package *rp; -}; #define power_zone_to_rapl_domain(_zone) \ container_of(_zone, struct rapl_domain, power_zone) -/* maximum rapl package domain name: package-%d-die-%d */ -#define PACKAGE_DOMAIN_NAME_LENGTH 30 - - -/* Each rapl package contains multiple domains, these are the common - * data across RAPL domains within a package. - */ -struct rapl_package { - unsigned int id; /* logical die id, equals physical 1-die systems */ - unsigned int nr_domains; - unsigned long domain_map; /* bit map of active domains */ - unsigned int power_unit; - unsigned int energy_unit; - unsigned int time_unit; - struct rapl_domain *domains; /* array of domains, sized at runtime */ - struct powercap_zone *power_zone; /* keep track of parent zone */ - unsigned long power_limit_irq; /* keep track of package power limit - * notify interrupt enable status. - */ - struct list_head plist; - int lead_cpu; /* one active cpu per package for access */ - /* Track active cpus */ - struct cpumask cpumask; - char name[PACKAGE_DOMAIN_NAME_LENGTH]; -}; - struct rapl_defaults { u8 floor_freq_reg_addr; int (*check_unit)(struct rapl_package *rp, int cpu); void (*set_floor_freq)(struct rapl_domain *rd, bool mode); u64 (*compute_time_window)(struct rapl_package *rp, u64 val, - bool to_raw); + bool to_raw); unsigned int dram_domain_energy_unit; }; static struct rapl_defaults *rapl_defaults; @@ -216,7 +113,7 @@ struct rapl_primitive_info { const char *name; u64 mask; int shift; - enum rapl_domain_msr_id id; + enum rapl_domain_reg_id id; enum unit_type unit; u32 flag; }; @@ -232,19 +129,18 @@ struct rapl_primitive_info { static void rapl_init_domains(struct rapl_package *rp); static int rapl_read_data_raw(struct rapl_domain *rd, - enum rapl_primitives prim, - bool xlate, u64 *data); + enum rapl_primitives prim, + bool xlate, u64 *data); static int rapl_write_data_raw(struct rapl_domain *rd, - enum rapl_primitives prim, - unsigned long long value); + enum rapl_primitives prim, + unsigned long long value); static u64 rapl_unit_xlate(struct rapl_domain *rd, - enum unit_type type, u64 value, - int to_raw); + enum unit_type type, u64 value, int to_raw); static void package_power_limit_irq_save(struct rapl_package *rp); -static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */ +static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */ -static const char * const rapl_domain_names[] = { +static const char *const rapl_domain_names[] = { "package", "core", "uncore", @@ -252,24 +148,8 @@ static const char * const rapl_domain_names[] = { "psys", }; -static struct powercap_control_type *control_type; /* PowerCap Controller */ -static struct rapl_domain *platform_rapl_domain; /* Platform (PSys) domain */ - -/* caller to ensure CPU hotplug lock is held */ -static struct rapl_package *rapl_find_package_domain(int cpu) -{ - int id = topology_logical_die_id(cpu); - struct rapl_package *rp; - - list_for_each_entry(rp, &rapl_packages, plist) { - if (rp->id == id) - return rp; - } - - return NULL; -} - -static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw) +static int get_energy_counter(struct powercap_zone *power_zone, + u64 *energy_raw) { struct rapl_domain *rd; u64 energy_now; @@ -368,50 +248,49 @@ static int get_domain_enable(struct powercap_zone *power_zone, bool *mode) static const struct powercap_zone_ops zone_ops[] = { /* RAPL_DOMAIN_PACKAGE */ { - .get_energy_uj = get_energy_counter, - .get_max_energy_range_uj = get_max_energy_counter, - .release = release_zone, - .set_enable = set_domain_enable, - .get_enable = get_domain_enable, - }, + .get_energy_uj = get_energy_counter, + .get_max_energy_range_uj = get_max_energy_counter, + .release = release_zone, + .set_enable = set_domain_enable, + .get_enable = get_domain_enable, + }, /* RAPL_DOMAIN_PP0 */ { - .get_energy_uj = get_energy_counter, - .get_max_energy_range_uj = get_max_energy_counter, - .release = release_zone, - .set_enable = set_domain_enable, - .get_enable = get_domain_enable, - }, + .get_energy_uj = get_energy_counter, + .get_max_energy_range_uj = get_max_energy_counter, + .release = release_zone, + .set_enable = set_domain_enable, + .get_enable = get_domain_enable, + }, /* RAPL_DOMAIN_PP1 */ { - .get_energy_uj = get_energy_counter, - .get_max_energy_range_uj = get_max_energy_counter, - .release = release_zone, - .set_enable = set_domain_enable, - .get_enable = get_domain_enable, - }, + .get_energy_uj = get_energy_counter, + .get_max_energy_range_uj = get_max_energy_counter, + .release = release_zone, + .set_enable = set_domain_enable, + .get_enable = get_domain_enable, + }, /* RAPL_DOMAIN_DRAM */ { - .get_energy_uj = get_energy_counter, - .get_max_energy_range_uj = get_max_energy_counter, - .release = release_zone, - .set_enable = set_domain_enable, - .get_enable = get_domain_enable, - }, + .get_energy_uj = get_energy_counter, + .get_max_energy_range_uj = get_max_energy_counter, + .release = release_zone, + .set_enable = set_domain_enable, + .get_enable = get_domain_enable, + }, /* RAPL_DOMAIN_PLATFORM */ { - .get_energy_uj = get_energy_counter, - .get_max_energy_range_uj = get_max_energy_counter, - .release = release_zone, - .set_enable = set_domain_enable, - .get_enable = get_domain_enable, - }, + .get_energy_uj = get_energy_counter, + .get_max_energy_range_uj = get_max_energy_counter, + .release = release_zone, + .set_enable = set_domain_enable, + .get_enable = get_domain_enable, + }, }; - /* * Constraint index used by powercap can be different than power limit (PL) - * index in that some PLs maybe missing due to non-existant MSRs. So we + * index in that some PLs maybe missing due to non-existent MSRs. So we * need to convert here by finding the valid PLs only (name populated). */ static int contraint_to_pl(struct rapl_domain *rd, int cid) @@ -430,7 +309,7 @@ static int contraint_to_pl(struct rapl_domain *rd, int cid) } static int set_power_limit(struct powercap_zone *power_zone, int cid, - u64 power_limit) + u64 power_limit) { struct rapl_domain *rd; struct rapl_package *rp; @@ -448,8 +327,8 @@ static int set_power_limit(struct powercap_zone *power_zone, int cid, rp = rd->rp; if (rd->state & DOMAIN_STATE_BIOS_LOCKED) { - dev_warn(&power_zone->dev, "%s locked by BIOS, monitoring only\n", - rd->name); + dev_warn(&power_zone->dev, + "%s locked by BIOS, monitoring only\n", rd->name); ret = -EACCES; goto set_exit; } @@ -472,7 +351,7 @@ set_exit: } static int get_current_power_limit(struct powercap_zone *power_zone, int cid, - u64 *data) + u64 *data) { struct rapl_domain *rd; u64 val; @@ -511,7 +390,7 @@ get_exit: } static int set_time_window(struct powercap_zone *power_zone, int cid, - u64 window) + u64 window) { struct rapl_domain *rd; int ret = 0; @@ -541,7 +420,8 @@ set_time_exit: return ret; } -static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data) +static int get_time_window(struct powercap_zone *power_zone, int cid, + u64 *data) { struct rapl_domain *rd; u64 val; @@ -576,7 +456,8 @@ get_time_exit: return ret; } -static const char *get_constraint_name(struct powercap_zone *power_zone, int cid) +static const char *get_constraint_name(struct powercap_zone *power_zone, + int cid) { struct rapl_domain *rd; int id; @@ -589,9 +470,7 @@ static const char *get_constraint_name(struct powercap_zone *power_zone, int cid return NULL; } - -static int get_max_power(struct powercap_zone *power_zone, int id, - u64 *data) +static int get_max_power(struct powercap_zone *power_zone, int id, u64 *data) { struct rapl_domain *rd; u64 val; @@ -633,73 +512,43 @@ static const struct powercap_zone_constraint_ops constraint_ops = { /* called after domain detection and package level data are set */ static void rapl_init_domains(struct rapl_package *rp) { - int i; + enum rapl_domain_type i; + enum rapl_domain_reg_id j; struct rapl_domain *rd = rp->domains; for (i = 0; i < RAPL_DOMAIN_MAX; i++) { unsigned int mask = rp->domain_map & (1 << i); - switch (mask) { - case BIT(RAPL_DOMAIN_PACKAGE): - rd->name = rapl_domain_names[RAPL_DOMAIN_PACKAGE]; - rd->id = RAPL_DOMAIN_PACKAGE; - rd->msrs[0] = MSR_PKG_POWER_LIMIT; - rd->msrs[1] = MSR_PKG_ENERGY_STATUS; - rd->msrs[2] = MSR_PKG_PERF_STATUS; - rd->msrs[3] = 0; - rd->msrs[4] = MSR_PKG_POWER_INFO; - rd->rpl[0].prim_id = PL1_ENABLE; - rd->rpl[0].name = pl1_name; + + if (!mask) + continue; + + rd->rp = rp; + rd->name = rapl_domain_names[i]; + rd->id = i; + rd->rpl[0].prim_id = PL1_ENABLE; + rd->rpl[0].name = pl1_name; + /* some domain may support two power limits */ + if (rp->priv->limits[i] == 2) { rd->rpl[1].prim_id = PL2_ENABLE; rd->rpl[1].name = pl2_name; - break; - case BIT(RAPL_DOMAIN_PP0): - rd->name = rapl_domain_names[RAPL_DOMAIN_PP0]; - rd->id = RAPL_DOMAIN_PP0; - rd->msrs[0] = MSR_PP0_POWER_LIMIT; - rd->msrs[1] = MSR_PP0_ENERGY_STATUS; - rd->msrs[2] = 0; - rd->msrs[3] = MSR_PP0_POLICY; - rd->msrs[4] = 0; - rd->rpl[0].prim_id = PL1_ENABLE; - rd->rpl[0].name = pl1_name; - break; - case BIT(RAPL_DOMAIN_PP1): - rd->name = rapl_domain_names[RAPL_DOMAIN_PP1]; - rd->id = RAPL_DOMAIN_PP1; - rd->msrs[0] = MSR_PP1_POWER_LIMIT; - rd->msrs[1] = MSR_PP1_ENERGY_STATUS; - rd->msrs[2] = 0; - rd->msrs[3] = MSR_PP1_POLICY; - rd->msrs[4] = 0; - rd->rpl[0].prim_id = PL1_ENABLE; - rd->rpl[0].name = pl1_name; - break; - case BIT(RAPL_DOMAIN_DRAM): - rd->name = rapl_domain_names[RAPL_DOMAIN_DRAM]; - rd->id = RAPL_DOMAIN_DRAM; - rd->msrs[0] = MSR_DRAM_POWER_LIMIT; - rd->msrs[1] = MSR_DRAM_ENERGY_STATUS; - rd->msrs[2] = MSR_DRAM_PERF_STATUS; - rd->msrs[3] = 0; - rd->msrs[4] = MSR_DRAM_POWER_INFO; - rd->rpl[0].prim_id = PL1_ENABLE; - rd->rpl[0].name = pl1_name; + } + + for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++) + rd->regs[j] = rp->priv->regs[i][j]; + + if (i == RAPL_DOMAIN_DRAM) { rd->domain_energy_unit = - rapl_defaults->dram_domain_energy_unit; + rapl_defaults->dram_domain_energy_unit; if (rd->domain_energy_unit) pr_info("DRAM domain energy unit %dpj\n", rd->domain_energy_unit); - break; - } - if (mask) { - rd->rp = rp; - rd++; } + rd++; } } static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, - u64 value, int to_raw) + u64 value, int to_raw) { u64 units = 1; struct rapl_package *rp = rd->rp; @@ -736,40 +585,40 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, static struct rapl_primitive_info rpi[] = { /* name, mask, shift, msr index, unit divisor */ PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0, - RAPL_DOMAIN_MSR_STATUS, ENERGY_UNIT, 0), + RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0), PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0, - RAPL_DOMAIN_MSR_LIMIT, POWER_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32, - RAPL_DOMAIN_MSR_LIMIT, POWER_UNIT, 0), - PRIMITIVE_INFO_INIT(FW_LOCK, POWER_PP_LOCK, 31, - RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0), + PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31, + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15, - RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16, - RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47, - RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48, - RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0), PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17, - RAPL_DOMAIN_MSR_LIMIT, TIME_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49, - RAPL_DOMAIN_MSR_LIMIT, TIME_UNIT, 0), + RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0), PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK, - 0, RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0), + 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32, - RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0), + RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16, - RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0), + RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0), PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48, - RAPL_DOMAIN_MSR_INFO, TIME_UNIT, 0), + RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0), PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0, - RAPL_DOMAIN_MSR_PERF, TIME_UNIT, 0), + RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0), PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0, - RAPL_DOMAIN_MSR_POLICY, ARBITRARY_UNIT, 0), + RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0), /* non-hardware */ PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT, - RAPL_PRIMITIVE_DERIVED), + RAPL_PRIMITIVE_DERIVED), {NULL, 0, 0, 0}, }; @@ -787,26 +636,25 @@ static struct rapl_primitive_info rpi[] = { * 63-------------------------- 31--------------------------- 0 */ static int rapl_read_data_raw(struct rapl_domain *rd, - enum rapl_primitives prim, - bool xlate, u64 *data) + enum rapl_primitives prim, bool xlate, u64 *data) { - u64 value, final; - u32 msr; + u64 value; struct rapl_primitive_info *rp = &rpi[prim]; + struct reg_action ra; int cpu; if (!rp->name || rp->flag & RAPL_PRIMITIVE_DUMMY) return -EINVAL; - msr = rd->msrs[rp->id]; - if (!msr) + ra.reg = rd->regs[rp->id]; + if (!ra.reg) return -EINVAL; cpu = rd->rp->lead_cpu; - /* special-case package domain, which uses a different bit*/ - if (prim == FW_LOCK && rd->id == RAPL_DOMAIN_PACKAGE) { - rp->mask = POWER_PACKAGE_LOCK; + /* domain with 2 limits has different bit */ + if (prim == FW_LOCK && rd->rp->priv->limits[rd->id] == 2) { + rp->mask = POWER_HIGH_LOCK; rp->shift = 63; } /* non-hardware data are collected by the polling thread */ @@ -815,56 +663,32 @@ static int rapl_read_data_raw(struct rapl_domain *rd, return 0; } - if (rdmsrl_safe_on_cpu(cpu, msr, &value)) { - pr_debug("failed to read msr 0x%x on cpu %d\n", msr, cpu); + ra.mask = rp->mask; + + if (rd->rp->priv->read_raw(cpu, &ra)) { + pr_debug("failed to read reg 0x%llx on cpu %d\n", ra.reg, cpu); return -EIO; } - final = value & rp->mask; - final = final >> rp->shift; + value = ra.value >> rp->shift; + if (xlate) - *data = rapl_unit_xlate(rd, rp->unit, final, 0); + *data = rapl_unit_xlate(rd, rp->unit, value, 0); else - *data = final; + *data = value; return 0; } - -static int msrl_update_safe(u32 msr_no, u64 clear_mask, u64 set_mask) -{ - int err; - u64 val; - - err = rdmsrl_safe(msr_no, &val); - if (err) - goto out; - - val &= ~clear_mask; - val |= set_mask; - - err = wrmsrl_safe(msr_no, val); - -out: - return err; -} - -static void msrl_update_func(void *info) -{ - struct msrl_action *ma = info; - - ma->err = msrl_update_safe(ma->msr_no, ma->clear_mask, ma->set_mask); -} - /* Similar use of primitive info in the read counterpart */ static int rapl_write_data_raw(struct rapl_domain *rd, - enum rapl_primitives prim, - unsigned long long value) + enum rapl_primitives prim, + unsigned long long value) { struct rapl_primitive_info *rp = &rpi[prim]; int cpu; u64 bits; - struct msrl_action ma; + struct reg_action ra; int ret; cpu = rd->rp->lead_cpu; @@ -872,17 +696,13 @@ static int rapl_write_data_raw(struct rapl_domain *rd, bits <<= rp->shift; bits &= rp->mask; - memset(&ma, 0, sizeof(ma)); + memset(&ra, 0, sizeof(ra)); - ma.msr_no = rd->msrs[rp->id]; - ma.clear_mask = rp->mask; - ma.set_mask = bits; + ra.reg = rd->regs[rp->id]; + ra.mask = rp->mask; + ra.value = bits; - ret = smp_call_function_single(cpu, msrl_update_func, &ma, 1); - if (ret) - WARN_ON_ONCE(ret); - else - ret = ma.err; + ret = rd->rp->priv->write_raw(cpu, &ra); return ret; } @@ -900,51 +720,56 @@ static int rapl_write_data_raw(struct rapl_domain *rd, */ static int rapl_check_unit_core(struct rapl_package *rp, int cpu) { - u64 msr_val; + struct reg_action ra; u32 value; - if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) { - pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n", - MSR_RAPL_POWER_UNIT, cpu); + ra.reg = rp->priv->reg_unit; + ra.mask = ~0; + if (rp->priv->read_raw(cpu, &ra)) { + pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", + rp->priv->reg_unit, cpu); return -ENODEV; } - value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; + value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; rp->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); - value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; + value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; rp->power_unit = 1000000 / (1 << value); - value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; + value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; rp->time_unit = 1000000 / (1 << value); pr_debug("Core CPU %s energy=%dpJ, time=%dus, power=%duW\n", - rp->name, rp->energy_unit, rp->time_unit, rp->power_unit); + rp->name, rp->energy_unit, rp->time_unit, rp->power_unit); return 0; } static int rapl_check_unit_atom(struct rapl_package *rp, int cpu) { - u64 msr_val; + struct reg_action ra; u32 value; - if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) { - pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n", - MSR_RAPL_POWER_UNIT, cpu); + ra.reg = rp->priv->reg_unit; + ra.mask = ~0; + if (rp->priv->read_raw(cpu, &ra)) { + pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n", + rp->priv->reg_unit, cpu); return -ENODEV; } - value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; + + value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; rp->energy_unit = ENERGY_UNIT_SCALE * 1 << value; - value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; + value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; rp->power_unit = (1 << value) * 1000; - value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; + value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; rp->time_unit = 1000000 / (1 << value); pr_debug("Atom %s energy=%dpJ, time=%dus, power=%duW\n", - rp->name, rp->energy_unit, rp->time_unit, rp->power_unit); + rp->name, rp->energy_unit, rp->time_unit, rp->power_unit); return 0; } @@ -964,7 +789,6 @@ static void power_limit_irq_save_cpu(void *info) wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); } - /* REVISIT: * When package power limit is set artificially low by RAPL, LVT * thermal interrupt for package power limit should be ignored @@ -1048,9 +872,9 @@ static void set_floor_freq_atom(struct rapl_domain *rd, bool enable) } static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, - bool to_raw) + bool to_raw) { - u64 f, y; /* fraction and exp. used for time unit */ + u64 f, y; /* fraction and exp. used for time unit */ /* * Special processing based on 2^Y*(1+F/4), refer @@ -1070,7 +894,7 @@ static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, } static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value, - bool to_raw) + bool to_raw) { /* * Atom time unit encoding is straight forward val * time_unit, @@ -1078,8 +902,8 @@ static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value, */ if (!to_raw) return (value) ? value *= rp->time_unit : rp->time_unit; - else - value = div64_u64(value, rp->time_unit); + + value = div64_u64(value, rp->time_unit); return value; } @@ -1127,43 +951,48 @@ static const struct rapl_defaults rapl_defaults_cht = { }; static const struct x86_cpu_id rapl_ids[] __initconst = { - INTEL_CPU_FAM6(SANDYBRIDGE, rapl_defaults_core), - INTEL_CPU_FAM6(SANDYBRIDGE_X, rapl_defaults_core), - - INTEL_CPU_FAM6(IVYBRIDGE, rapl_defaults_core), - INTEL_CPU_FAM6(IVYBRIDGE_X, rapl_defaults_core), - - INTEL_CPU_FAM6(HASWELL_CORE, rapl_defaults_core), - INTEL_CPU_FAM6(HASWELL_ULT, rapl_defaults_core), - INTEL_CPU_FAM6(HASWELL_GT3E, rapl_defaults_core), - INTEL_CPU_FAM6(HASWELL_X, rapl_defaults_hsw_server), - - INTEL_CPU_FAM6(BROADWELL_CORE, rapl_defaults_core), - INTEL_CPU_FAM6(BROADWELL_GT3E, rapl_defaults_core), - INTEL_CPU_FAM6(BROADWELL_XEON_D, rapl_defaults_core), - INTEL_CPU_FAM6(BROADWELL_X, rapl_defaults_hsw_server), - - INTEL_CPU_FAM6(SKYLAKE_DESKTOP, rapl_defaults_core), - INTEL_CPU_FAM6(SKYLAKE_MOBILE, rapl_defaults_core), - INTEL_CPU_FAM6(SKYLAKE_X, rapl_defaults_hsw_server), - INTEL_CPU_FAM6(KABYLAKE_MOBILE, rapl_defaults_core), - INTEL_CPU_FAM6(KABYLAKE_DESKTOP, rapl_defaults_core), - INTEL_CPU_FAM6(CANNONLAKE_MOBILE, rapl_defaults_core), - INTEL_CPU_FAM6(ICELAKE_MOBILE, rapl_defaults_core), - - INTEL_CPU_FAM6(ATOM_SILVERMONT, rapl_defaults_byt), - INTEL_CPU_FAM6(ATOM_AIRMONT, rapl_defaults_cht), - INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, rapl_defaults_tng), - INTEL_CPU_FAM6(ATOM_AIRMONT_MID, rapl_defaults_ann), - INTEL_CPU_FAM6(ATOM_GOLDMONT, rapl_defaults_core), - INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, rapl_defaults_core), - INTEL_CPU_FAM6(ATOM_GOLDMONT_X, rapl_defaults_core), - INTEL_CPU_FAM6(ATOM_TREMONT_X, rapl_defaults_core), - - INTEL_CPU_FAM6(XEON_PHI_KNL, rapl_defaults_hsw_server), - INTEL_CPU_FAM6(XEON_PHI_KNM, rapl_defaults_hsw_server), + INTEL_CPU_FAM6(SANDYBRIDGE, rapl_defaults_core), + INTEL_CPU_FAM6(SANDYBRIDGE_X, rapl_defaults_core), + + INTEL_CPU_FAM6(IVYBRIDGE, rapl_defaults_core), + INTEL_CPU_FAM6(IVYBRIDGE_X, rapl_defaults_core), + + INTEL_CPU_FAM6(HASWELL_CORE, rapl_defaults_core), + INTEL_CPU_FAM6(HASWELL_ULT, rapl_defaults_core), + INTEL_CPU_FAM6(HASWELL_GT3E, rapl_defaults_core), + INTEL_CPU_FAM6(HASWELL_X, rapl_defaults_hsw_server), + + INTEL_CPU_FAM6(BROADWELL_CORE, rapl_defaults_core), + INTEL_CPU_FAM6(BROADWELL_GT3E, rapl_defaults_core), + INTEL_CPU_FAM6(BROADWELL_XEON_D, rapl_defaults_core), + INTEL_CPU_FAM6(BROADWELL_X, rapl_defaults_hsw_server), + + INTEL_CPU_FAM6(SKYLAKE_DESKTOP, rapl_defaults_core), + INTEL_CPU_FAM6(SKYLAKE_MOBILE, rapl_defaults_core), + INTEL_CPU_FAM6(SKYLAKE_X, rapl_defaults_hsw_server), + INTEL_CPU_FAM6(KABYLAKE_MOBILE, rapl_defaults_core), + INTEL_CPU_FAM6(KABYLAKE_DESKTOP, rapl_defaults_core), + INTEL_CPU_FAM6(CANNONLAKE_MOBILE, rapl_defaults_core), + INTEL_CPU_FAM6(ICELAKE_MOBILE, rapl_defaults_core), + INTEL_CPU_FAM6(ICELAKE_DESKTOP, rapl_defaults_core), + INTEL_CPU_FAM6(ICELAKE_NNPI, rapl_defaults_core), + INTEL_CPU_FAM6(ICELAKE_X, rapl_defaults_hsw_server), + INTEL_CPU_FAM6(ICELAKE_XEON_D, rapl_defaults_hsw_server), + + INTEL_CPU_FAM6(ATOM_SILVERMONT, rapl_defaults_byt), + INTEL_CPU_FAM6(ATOM_AIRMONT, rapl_defaults_cht), + INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, rapl_defaults_tng), + INTEL_CPU_FAM6(ATOM_AIRMONT_MID, rapl_defaults_ann), + INTEL_CPU_FAM6(ATOM_GOLDMONT, rapl_defaults_core), + INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, rapl_defaults_core), + INTEL_CPU_FAM6(ATOM_GOLDMONT_X, rapl_defaults_core), + INTEL_CPU_FAM6(ATOM_TREMONT_X, rapl_defaults_core), + + INTEL_CPU_FAM6(XEON_PHI_KNL, rapl_defaults_hsw_server), + INTEL_CPU_FAM6(XEON_PHI_KNM, rapl_defaults_hsw_server), {} }; + MODULE_DEVICE_TABLE(x86cpu, rapl_ids); /* Read once for all raw primitive data for domains */ @@ -1179,22 +1008,12 @@ static void rapl_update_domain_data(struct rapl_package *rp) for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) { if (!rapl_read_data_raw(&rp->domains[dmn], prim, rpi[prim].unit, &val)) - rp->domains[dmn].rdd.primitives[prim] = val; + rp->domains[dmn].rdd.primitives[prim] = val; } } } -static void rapl_unregister_powercap(void) -{ - if (platform_rapl_domain) { - powercap_unregister_zone(control_type, - &platform_rapl_domain->power_zone); - kfree(platform_rapl_domain); - } - powercap_unregister_control_type(control_type); -} - static int rapl_package_register_powercap(struct rapl_package *rp) { struct rapl_domain *rd; @@ -1204,20 +1023,18 @@ static int rapl_package_register_powercap(struct rapl_package *rp) /* Update the domain data of the new package */ rapl_update_domain_data(rp); - /* first we register package domain as the parent zone*/ + /* first we register package domain as the parent zone */ for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { if (rd->id == RAPL_DOMAIN_PACKAGE) { nr_pl = find_nr_power_limit(rd); pr_debug("register package domain %s\n", rp->name); power_zone = powercap_register_zone(&rd->power_zone, - control_type, - rp->name, NULL, - &zone_ops[rd->id], - nr_pl, - &constraint_ops); + rp->priv->control_type, rp->name, + NULL, &zone_ops[rd->id], nr_pl, + &constraint_ops); if (IS_ERR(power_zone)) { pr_debug("failed to register power zone %s\n", - rp->name); + rp->name); return PTR_ERR(power_zone); } /* track parent zone in per package/socket data */ @@ -1230,21 +1047,21 @@ static int rapl_package_register_powercap(struct rapl_package *rp) pr_err("no package domain found, unknown topology!\n"); return -ENODEV; } - /* now register domains as children of the socket/package*/ + /* now register domains as children of the socket/package */ for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { if (rd->id == RAPL_DOMAIN_PACKAGE) continue; /* number of power limits per domain varies */ nr_pl = find_nr_power_limit(rd); power_zone = powercap_register_zone(&rd->power_zone, - control_type, rd->name, - rp->power_zone, - &zone_ops[rd->id], nr_pl, - &constraint_ops); + rp->priv->control_type, + rd->name, rp->power_zone, + &zone_ops[rd->id], nr_pl, + &constraint_ops); if (IS_ERR(power_zone)) { pr_debug("failed to register power_zone, %s:%s\n", - rp->name, rd->name); + rp->name, rd->name); ret = PTR_ERR(power_zone); goto err_cleanup; } @@ -1258,22 +1075,30 @@ err_cleanup: */ while (--rd >= rp->domains) { pr_debug("unregister %s domain %s\n", rp->name, rd->name); - powercap_unregister_zone(control_type, &rd->power_zone); + powercap_unregister_zone(rp->priv->control_type, + &rd->power_zone); } return ret; } -static int __init rapl_register_psys(void) +int rapl_add_platform_domain(struct rapl_if_priv *priv) { struct rapl_domain *rd; struct powercap_zone *power_zone; - u64 val; + struct reg_action ra; + int ret; - if (rdmsrl_safe_on_cpu(0, MSR_PLATFORM_ENERGY_STATUS, &val) || !val) + ra.reg = priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_STATUS]; + ra.mask = ~0; + ret = priv->read_raw(0, &ra); + if (ret || !ra.value) return -ENODEV; - if (rdmsrl_safe_on_cpu(0, MSR_PLATFORM_POWER_LIMIT, &val) || !val) + ra.reg = priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_LIMIT]; + ra.mask = ~0; + ret = priv->read_raw(0, &ra); + if (ret || !ra.value) return -ENODEV; rd = kzalloc(sizeof(*rd), GFP_KERNEL); @@ -1282,15 +1107,17 @@ static int __init rapl_register_psys(void) rd->name = rapl_domain_names[RAPL_DOMAIN_PLATFORM]; rd->id = RAPL_DOMAIN_PLATFORM; - rd->msrs[0] = MSR_PLATFORM_POWER_LIMIT; - rd->msrs[1] = MSR_PLATFORM_ENERGY_STATUS; + rd->regs[RAPL_DOMAIN_REG_LIMIT] = + priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_LIMIT]; + rd->regs[RAPL_DOMAIN_REG_STATUS] = + priv->regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_STATUS]; rd->rpl[0].prim_id = PL1_ENABLE; rd->rpl[0].name = pl1_name; rd->rpl[1].prim_id = PL2_ENABLE; rd->rpl[1].name = pl2_name; - rd->rp = rapl_find_package_domain(0); + rd->rp = rapl_find_package_domain(0, priv); - power_zone = powercap_register_zone(&rd->power_zone, control_type, + power_zone = powercap_register_zone(&rd->power_zone, priv->control_type, "psys", NULL, &zone_ops[RAPL_DOMAIN_PLATFORM], 2, &constraint_ops); @@ -1300,38 +1127,32 @@ static int __init rapl_register_psys(void) return PTR_ERR(power_zone); } - platform_rapl_domain = rd; + priv->platform_rapl_domain = rd; return 0; } +EXPORT_SYMBOL_GPL(rapl_add_platform_domain); -static int __init rapl_register_powercap(void) +void rapl_remove_platform_domain(struct rapl_if_priv *priv) { - control_type = powercap_register_control_type(NULL, "intel-rapl", NULL); - if (IS_ERR(control_type)) { - pr_debug("failed to register powercap control_type.\n"); - return PTR_ERR(control_type); + if (priv->platform_rapl_domain) { + powercap_unregister_zone(priv->control_type, + &priv->platform_rapl_domain->power_zone); + kfree(priv->platform_rapl_domain); } - return 0; } +EXPORT_SYMBOL_GPL(rapl_remove_platform_domain); -static int rapl_check_domain(int cpu, int domain) +static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp) { - unsigned msr; - u64 val = 0; + struct reg_action ra; switch (domain) { case RAPL_DOMAIN_PACKAGE: - msr = MSR_PKG_ENERGY_STATUS; - break; case RAPL_DOMAIN_PP0: - msr = MSR_PP0_ENERGY_STATUS; - break; case RAPL_DOMAIN_PP1: - msr = MSR_PP1_ENERGY_STATUS; - break; case RAPL_DOMAIN_DRAM: - msr = MSR_DRAM_ENERGY_STATUS; + ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS]; break; case RAPL_DOMAIN_PLATFORM: /* PSYS(PLATFORM) is not a CPU domain, so avoid printng error */ @@ -1343,19 +1164,20 @@ static int rapl_check_domain(int cpu, int domain) /* make sure domain counters are available and contains non-zero * values, otherwise skip it. */ - if (rdmsrl_safe_on_cpu(cpu, msr, &val) || !val) + + ra.mask = ~0; + if (rp->priv->read_raw(cpu, &ra) || !ra.value) return -ENODEV; return 0; } - /* * Check if power limits are available. Two cases when they are not available: * 1. Locked by BIOS, in this case we still provide read-only access so that * users can see what limit is set by the BIOS. * 2. Some CPUs make some domains monitoring only which means PLx MSRs may not - * exist at all. In this case, we do not show the contraints in powercap. + * exist at all. In this case, we do not show the constraints in powercap. * * Called after domains are detected and initialized. */ @@ -1372,9 +1194,10 @@ static void rapl_detect_powerlimit(struct rapl_domain *rd) rd->state |= DOMAIN_STATE_BIOS_LOCKED; } } - /* check if power limit MSRs exists, otherwise domain is monitoring only */ + /* check if power limit MSR exists, otherwise domain is monitoring only */ for (i = 0; i < NR_POWER_LIMITS; i++) { int prim = rd->rpl[i].prim_id; + if (rapl_read_data_raw(rd, prim, false, &val64)) rd->rpl[i].name = NULL; } @@ -1390,12 +1213,12 @@ static int rapl_detect_domains(struct rapl_package *rp, int cpu) for (i = 0; i < RAPL_DOMAIN_MAX; i++) { /* use physical package id to read counters */ - if (!rapl_check_domain(cpu, i)) { + if (!rapl_check_domain(cpu, i, rp)) { rp->domain_map |= 1 << i; pr_info("Found RAPL domain %s\n", rapl_domain_names[i]); } } - rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX); + rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX); if (!rp->nr_domains) { pr_debug("no valid rapl domains found in %s\n", rp->name); return -ENODEV; @@ -1403,7 +1226,7 @@ static int rapl_detect_domains(struct rapl_package *rp, int cpu) pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name); rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain), - GFP_KERNEL); + GFP_KERNEL); if (!rp->domains) return -ENOMEM; @@ -1416,7 +1239,7 @@ static int rapl_detect_domains(struct rapl_package *rp, int cpu) } /* called from CPU hotplug notifier, hotplug lock held */ -static void rapl_remove_package(struct rapl_package *rp) +void rapl_remove_package(struct rapl_package *rp) { struct rapl_domain *rd, *rd_package = NULL; @@ -1435,16 +1258,35 @@ static void rapl_remove_package(struct rapl_package *rp) } pr_debug("remove package, undo power limit on %s: %s\n", rp->name, rd->name); - powercap_unregister_zone(control_type, &rd->power_zone); + powercap_unregister_zone(rp->priv->control_type, + &rd->power_zone); } /* do parent zone last */ - powercap_unregister_zone(control_type, &rd_package->power_zone); + powercap_unregister_zone(rp->priv->control_type, + &rd_package->power_zone); list_del(&rp->plist); kfree(rp); } +EXPORT_SYMBOL_GPL(rapl_remove_package); + +/* caller to ensure CPU hotplug lock is held */ +struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv) +{ + int id = topology_logical_die_id(cpu); + struct rapl_package *rp; + + list_for_each_entry(rp, &rapl_packages, plist) { + if (rp->id == id + && rp->priv->control_type == priv->control_type) + return rp; + } + + return NULL; +} +EXPORT_SYMBOL_GPL(rapl_find_package_domain); /* called from CPU hotplug notifier, hotplug lock held */ -static struct rapl_package *rapl_add_package(int cpu) +struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) { int id = topology_logical_die_id(cpu); struct rapl_package *rp; @@ -1458,17 +1300,17 @@ static struct rapl_package *rapl_add_package(int cpu) /* add the new package to the list */ rp->id = id; rp->lead_cpu = cpu; + rp->priv = priv; if (topology_max_die_per_package() > 1) snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, - "package-%d-die-%d", c->phys_proc_id, c->cpu_die_id); + "package-%d-die-%d", c->phys_proc_id, c->cpu_die_id); else snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", - c->phys_proc_id); + c->phys_proc_id); /* check if the package contains valid domains */ - if (rapl_detect_domains(rp, cpu) || - rapl_defaults->check_unit(rp, cpu)) { + if (rapl_detect_domains(rp, cpu) || rapl_defaults->check_unit(rp, cpu)) { ret = -ENODEV; goto err_free_package; } @@ -1484,47 +1326,7 @@ err_free_package: kfree(rp); return ERR_PTR(ret); } - -/* Handles CPU hotplug on multi-socket systems. - * If a CPU goes online as the first CPU of the physical package - * we add the RAPL package to the system. Similarly, when the last - * CPU of the package is removed, we remove the RAPL package and its - * associated domains. Cooling devices are handled accordingly at - * per-domain level. - */ -static int rapl_cpu_online(unsigned int cpu) -{ - struct rapl_package *rp; - - rp = rapl_find_package_domain(cpu); - if (!rp) { - rp = rapl_add_package(cpu); - if (IS_ERR(rp)) - return PTR_ERR(rp); - } - cpumask_set_cpu(cpu, &rp->cpumask); - return 0; -} - -static int rapl_cpu_down_prep(unsigned int cpu) -{ - struct rapl_package *rp; - int lead_cpu; - - rp = rapl_find_package_domain(cpu); - if (!rp) - return 0; - - cpumask_clear_cpu(cpu, &rp->cpumask); - lead_cpu = cpumask_first(&rp->cpumask); - if (lead_cpu >= nr_cpu_ids) - rapl_remove_package(rp); - else if (rp->lead_cpu == cpu) - rp->lead_cpu = lead_cpu; - return 0; -} - -static enum cpuhp_state pcap_rapl_online; +EXPORT_SYMBOL_GPL(rapl_add_package); static void power_limit_state_save(void) { @@ -1542,17 +1344,15 @@ static void power_limit_state_save(void) switch (rd->rpl[i].prim_id) { case PL1_ENABLE: ret = rapl_read_data_raw(rd, - POWER_LIMIT1, - true, - &rd->rpl[i].last_power_limit); + POWER_LIMIT1, true, + &rd->rpl[i].last_power_limit); if (ret) rd->rpl[i].last_power_limit = 0; break; case PL2_ENABLE: ret = rapl_read_data_raw(rd, - POWER_LIMIT2, - true, - &rd->rpl[i].last_power_limit); + POWER_LIMIT2, true, + &rd->rpl[i].last_power_limit); if (ret) rd->rpl[i].last_power_limit = 0; break; @@ -1578,15 +1378,13 @@ static void power_limit_state_restore(void) switch (rd->rpl[i].prim_id) { case PL1_ENABLE: if (rd->rpl[i].last_power_limit) - rapl_write_data_raw(rd, - POWER_LIMIT1, - rd->rpl[i].last_power_limit); + rapl_write_data_raw(rd, POWER_LIMIT1, + rd->rpl[i].last_power_limit); break; case PL2_ENABLE: if (rd->rpl[i].last_power_limit) - rapl_write_data_raw(rd, - POWER_LIMIT2, - rd->rpl[i].last_power_limit); + rapl_write_data_raw(rd, POWER_LIMIT2, + rd->rpl[i].last_power_limit); break; } } @@ -1595,7 +1393,7 @@ static void power_limit_state_restore(void) } static int rapl_pm_callback(struct notifier_block *nb, - unsigned long mode, void *_unused) + unsigned long mode, void *_unused) { switch (mode) { case PM_SUSPEND_PREPARE: @@ -1612,6 +1410,8 @@ static struct notifier_block rapl_pm_notifier = { .notifier_call = rapl_pm_callback, }; +static struct platform_device *rapl_msr_platdev; + static int __init rapl_init(void) { const struct x86_cpu_id *id; @@ -1620,50 +1420,43 @@ static int __init rapl_init(void) id = x86_match_cpu(rapl_ids); if (!id) { pr_err("driver does not support CPU family %d model %d\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); + boot_cpu_data.x86, boot_cpu_data.x86_model); return -ENODEV; } rapl_defaults = (struct rapl_defaults *)id->driver_data; - ret = rapl_register_powercap(); + ret = register_pm_notifier(&rapl_pm_notifier); if (ret) return ret; - ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online", - rapl_cpu_online, rapl_cpu_down_prep); - if (ret < 0) - goto err_unreg; - pcap_rapl_online = ret; - - /* Don't bail out if PSys is not supported */ - rapl_register_psys(); + rapl_msr_platdev = platform_device_alloc("intel_rapl_msr", 0); + if (!rapl_msr_platdev) { + ret = -ENOMEM; + goto end; + } - ret = register_pm_notifier(&rapl_pm_notifier); + ret = platform_device_add(rapl_msr_platdev); if (ret) - goto err_unreg_all; + platform_device_put(rapl_msr_platdev); - return 0; - -err_unreg_all: - cpuhp_remove_state(pcap_rapl_online); +end: + if (ret) + unregister_pm_notifier(&rapl_pm_notifier); -err_unreg: - rapl_unregister_powercap(); return ret; } static void __exit rapl_exit(void) { + platform_device_unregister(rapl_msr_platdev); unregister_pm_notifier(&rapl_pm_notifier); - cpuhp_remove_state(pcap_rapl_online); - rapl_unregister_powercap(); } module_init(rapl_init); module_exit(rapl_exit); -MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit)"); +MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code"); MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c new file mode 100644 index 000000000000..d5487965bdfe --- /dev/null +++ b/drivers/powercap/intel_rapl_msr.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Running Average Power Limit (RAPL) Driver via MSR interface + * Copyright (c) 2019, Intel Corporation. + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/list.h> +#include <linux/types.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/log2.h> +#include <linux/bitmap.h> +#include <linux/delay.h> +#include <linux/sysfs.h> +#include <linux/cpu.h> +#include <linux/powercap.h> +#include <linux/suspend.h> +#include <linux/intel_rapl.h> +#include <linux/processor.h> +#include <linux/platform_device.h> + +#include <asm/iosf_mbi.h> +#include <asm/cpu_device_id.h> +#include <asm/intel-family.h> + +/* Local defines */ +#define MSR_PLATFORM_POWER_LIMIT 0x0000065C + +/* private data for RAPL MSR Interface */ +static struct rapl_if_priv rapl_msr_priv = { + .reg_unit = MSR_RAPL_POWER_UNIT, + .regs[RAPL_DOMAIN_PACKAGE] = { + MSR_PKG_POWER_LIMIT, MSR_PKG_ENERGY_STATUS, MSR_PKG_PERF_STATUS, 0, MSR_PKG_POWER_INFO }, + .regs[RAPL_DOMAIN_PP0] = { + MSR_PP0_POWER_LIMIT, MSR_PP0_ENERGY_STATUS, 0, MSR_PP0_POLICY, 0 }, + .regs[RAPL_DOMAIN_PP1] = { + MSR_PP1_POWER_LIMIT, MSR_PP1_ENERGY_STATUS, 0, MSR_PP1_POLICY, 0 }, + .regs[RAPL_DOMAIN_DRAM] = { + MSR_DRAM_POWER_LIMIT, MSR_DRAM_ENERGY_STATUS, MSR_DRAM_PERF_STATUS, 0, MSR_DRAM_POWER_INFO }, + .regs[RAPL_DOMAIN_PLATFORM] = { + MSR_PLATFORM_POWER_LIMIT, MSR_PLATFORM_ENERGY_STATUS, 0, 0, 0}, + .limits[RAPL_DOMAIN_PACKAGE] = 2, +}; + +/* Handles CPU hotplug on multi-socket systems. + * If a CPU goes online as the first CPU of the physical package + * we add the RAPL package to the system. Similarly, when the last + * CPU of the package is removed, we remove the RAPL package and its + * associated domains. Cooling devices are handled accordingly at + * per-domain level. + */ +static int rapl_cpu_online(unsigned int cpu) +{ + struct rapl_package *rp; + + rp = rapl_find_package_domain(cpu, &rapl_msr_priv); + if (!rp) { + rp = rapl_add_package(cpu, &rapl_msr_priv); + if (IS_ERR(rp)) + return PTR_ERR(rp); + } + cpumask_set_cpu(cpu, &rp->cpumask); + return 0; +} + +static int rapl_cpu_down_prep(unsigned int cpu) +{ + struct rapl_package *rp; + int lead_cpu; + + rp = rapl_find_package_domain(cpu, &rapl_msr_priv); + if (!rp) + return 0; + + cpumask_clear_cpu(cpu, &rp->cpumask); + lead_cpu = cpumask_first(&rp->cpumask); + if (lead_cpu >= nr_cpu_ids) + rapl_remove_package(rp); + else if (rp->lead_cpu == cpu) + rp->lead_cpu = lead_cpu; + return 0; +} + +static int rapl_msr_read_raw(int cpu, struct reg_action *ra) +{ + u32 msr = (u32)ra->reg; + + if (rdmsrl_safe_on_cpu(cpu, msr, &ra->value)) { + pr_debug("failed to read msr 0x%x on cpu %d\n", msr, cpu); + return -EIO; + } + ra->value &= ra->mask; + return 0; +} + +static void rapl_msr_update_func(void *info) +{ + struct reg_action *ra = info; + u32 msr = (u32)ra->reg; + u64 val; + + ra->err = rdmsrl_safe(msr, &val); + if (ra->err) + return; + + val &= ~ra->mask; + val |= ra->value; + + ra->err = wrmsrl_safe(msr, val); +} + +static int rapl_msr_write_raw(int cpu, struct reg_action *ra) +{ + int ret; + + ret = smp_call_function_single(cpu, rapl_msr_update_func, ra, 1); + if (WARN_ON_ONCE(ret)) + return ret; + + return ra->err; +} + +static int rapl_msr_probe(struct platform_device *pdev) +{ + int ret; + + rapl_msr_priv.read_raw = rapl_msr_read_raw; + rapl_msr_priv.write_raw = rapl_msr_write_raw; + + rapl_msr_priv.control_type = powercap_register_control_type(NULL, "intel-rapl", NULL); + if (IS_ERR(rapl_msr_priv.control_type)) { + pr_debug("failed to register powercap control_type.\n"); + return PTR_ERR(rapl_msr_priv.control_type); + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online", + rapl_cpu_online, rapl_cpu_down_prep); + if (ret < 0) + goto out; + rapl_msr_priv.pcap_rapl_online = ret; + + /* Don't bail out if PSys is not supported */ + rapl_add_platform_domain(&rapl_msr_priv); + + return 0; + +out: + if (ret) + powercap_unregister_control_type(rapl_msr_priv.control_type); + return ret; +} + +static int rapl_msr_remove(struct platform_device *pdev) +{ + cpuhp_remove_state(rapl_msr_priv.pcap_rapl_online); + rapl_remove_platform_domain(&rapl_msr_priv); + powercap_unregister_control_type(rapl_msr_priv.control_type); + return 0; +} + +static const struct platform_device_id rapl_msr_ids[] = { + { .name = "intel_rapl_msr", }, + {} +}; +MODULE_DEVICE_TABLE(platform, rapl_msr_ids); + +static struct platform_driver intel_rapl_msr_driver = { + .probe = rapl_msr_probe, + .remove = rapl_msr_remove, + .id_table = rapl_msr_ids, + .driver = { + .name = "intel_rapl_msr", + }, +}; + +module_platform_driver(intel_rapl_msr_driver); + +MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit) control via MSR interface"); +MODULE_AUTHOR("Zhang Rui <rui.zhang@intel.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c index 3a546ec10d90..22a65ad4e46e 100644 --- a/drivers/pps/pps.c +++ b/drivers/pps/pps.c @@ -152,6 +152,14 @@ static long pps_cdev_ioctl(struct file *file, pps->params.mode |= PPS_CANWAIT; pps->params.api_version = PPS_API_VERS; + /* + * Clear unused fields of pps_kparams to avoid leaking + * uninitialized data of the PPS_SETPARAMS caller via + * PPS_GETPARAMS + */ + pps->params.assert_off_tu.flags = 0; + pps->params.clear_off_tu.flags = 0; + spin_unlock_irq(&pps->lock); break; diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c index ce7a90e68042..8155f59ece38 100644 --- a/drivers/rapidio/devices/rio_mport_cdev.c +++ b/drivers/rapidio/devices/rio_mport_cdev.c @@ -1686,6 +1686,7 @@ static int rio_mport_add_riodev(struct mport_cdev_priv *priv, if (copy_from_user(&dev_info, arg, sizeof(dev_info))) return -EFAULT; + dev_info.name[sizeof(dev_info.name) - 1] = '\0'; rmcd_debug(RDEV, "name:%s ct:0x%x did:0x%x hc:0x%x", dev_info.name, dev_info.comptag, dev_info.destid, dev_info.hopcount); @@ -1817,6 +1818,7 @@ static int rio_mport_del_riodev(struct mport_cdev_priv *priv, void __user *arg) if (copy_from_user(&dev_info, arg, sizeof(dev_info))) return -EFAULT; + dev_info.name[sizeof(dev_info.name) - 1] = '\0'; mport = priv->md->mport; diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index 18be41b8aa7e..28ed306982f7 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -16,7 +16,7 @@ if REMOTEPROC config IMX_REMOTEPROC tristate "IMX6/7 remoteproc support" - depends on SOC_IMX6SX || SOC_IMX7D + depends on ARCH_MXC help Say y here to support iMX's remote processors (Cortex M4 on iMX7D) via the remote processor framework. @@ -116,6 +116,7 @@ config QCOM_Q6V5_MSS depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n select MFD_SYSCON + select QCOM_MDT_LOADER select QCOM_Q6V5_COMMON select QCOM_RPROC_COMMON select QCOM_SCM @@ -198,6 +199,21 @@ config ST_REMOTEPROC config ST_SLIM_REMOTEPROC tristate +config STM32_RPROC + tristate "STM32 remoteproc support" + depends on ARCH_STM32 + depends on REMOTEPROC + select MAILBOX + help + Say y here to support STM32 MCU processors via the + remote processor framework. + + You want to say y here in order to enable AMP + use-cases to run on your platform (dedicated firmware could be + offloaded to remote MCU processors using this framework). + + This can be either built-in or a loadable module. + endif # REMOTEPROC endmenu diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile index ce5d061e92be..00f09e658cb3 100644 --- a/drivers/remoteproc/Makefile +++ b/drivers/remoteproc/Makefile @@ -26,3 +26,4 @@ qcom_wcnss_pil-y += qcom_wcnss.o qcom_wcnss_pil-y += qcom_wcnss_iris.o obj-$(CONFIG_ST_REMOTEPROC) += st_remoteproc.o obj-$(CONFIG_ST_SLIM_REMOTEPROC) += st_slim_rproc.o +obj-$(CONFIG_STM32_RPROC) += stm32_rproc.o diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 36f2f14dad0c..3e72b6f38d4b 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -165,7 +165,7 @@ static int imx_rproc_start(struct rproc *rproc) ret = regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, dcfg->src_start); if (ret) - dev_err(dev, "Filed to enable M4!\n"); + dev_err(dev, "Failed to enable M4!\n"); return ret; } @@ -180,7 +180,7 @@ static int imx_rproc_stop(struct rproc *rproc) ret = regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, dcfg->src_stop); if (ret) - dev_err(dev, "Filed to stop M4!\n"); + dev_err(dev, "Failed to stop M4!\n"); return ret; } @@ -203,7 +203,7 @@ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da, } } - dev_warn(priv->dev, "Translation filed: da = 0x%llx len = 0x%x\n", + dev_warn(priv->dev, "Translation failed: da = 0x%llx len = 0x%x\n", da, len); return -ENOENT; } @@ -349,7 +349,7 @@ static int imx_rproc_probe(struct platform_device *pdev) ret = imx_rproc_addr_init(priv, pdev); if (ret) { - dev_err(dev, "filed on imx_rproc_addr_init\n"); + dev_err(dev, "failed on imx_rproc_addr_init\n"); goto err_put_rproc; } diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index 1f3ef9ee493c..e953886b2eb7 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -46,11 +46,9 @@ #define LPASS_PWR_ON_REG 0x10 #define LPASS_HALTREQ_REG 0x0 -/* list of clocks required by ADSP PIL */ -static const char * const adsp_clk_id[] = { - "sway_cbcr", "lpass_ahbs_aon_cbcr", "lpass_ahbm_aon_cbcr", - "qdsp6ss_xo", "qdsp6ss_sleep", "qdsp6ss_core", -}; +#define QDSP6SS_XO_CBCR 0x38 +#define QDSP6SS_CORE_CBCR 0x20 +#define QDSP6SS_SLEEP_CBCR 0x3c struct adsp_pil_data { int crash_reason_smem; @@ -59,6 +57,9 @@ struct adsp_pil_data { const char *ssr_name; const char *sysmon_name; int ssctl_id; + + const char **clk_ids; + int num_clks; }; struct qcom_adsp { @@ -75,7 +76,7 @@ struct qcom_adsp { void __iomem *qdsp6ss_base; struct reset_control *pdc_sync_reset; - struct reset_control *cc_lpass_restart; + struct reset_control *restart; struct regmap *halt_map; unsigned int halt_lpass; @@ -143,7 +144,7 @@ reset: /* Assert the LPASS PDC Reset */ reset_control_assert(adsp->pdc_sync_reset); /* Place the LPASS processor into reset */ - reset_control_assert(adsp->cc_lpass_restart); + reset_control_assert(adsp->restart); /* wait after asserting subsystem restart from AOSS */ usleep_range(200, 300); @@ -153,7 +154,7 @@ reset: /* De-assert the LPASS PDC Reset */ reset_control_deassert(adsp->pdc_sync_reset); /* Remove the LPASS reset */ - reset_control_deassert(adsp->cc_lpass_restart); + reset_control_deassert(adsp->restart); /* wait after de-asserting subsystem restart from AOSS */ usleep_range(200, 300); @@ -192,6 +193,15 @@ static int adsp_start(struct rproc *rproc) goto disable_power_domain; } + /* Enable the XO clock */ + writel(1, adsp->qdsp6ss_base + QDSP6SS_XO_CBCR); + + /* Enable the QDSP6SS sleep clock */ + writel(1, adsp->qdsp6ss_base + QDSP6SS_SLEEP_CBCR); + + /* Enable the QDSP6 core clock */ + writel(1, adsp->qdsp6ss_base + QDSP6SS_CORE_CBCR); + /* Program boot address */ writel(adsp->mem_phys >> 4, adsp->qdsp6ss_base + RST_EVB_REG); @@ -280,8 +290,9 @@ static const struct rproc_ops adsp_ops = { .load = adsp_load, }; -static int adsp_init_clock(struct qcom_adsp *adsp) +static int adsp_init_clock(struct qcom_adsp *adsp, const char **clk_ids) { + int num_clks = 0; int i, ret; adsp->xo = devm_clk_get(adsp->dev, "xo"); @@ -292,32 +303,39 @@ static int adsp_init_clock(struct qcom_adsp *adsp) return ret; } - adsp->num_clks = ARRAY_SIZE(adsp_clk_id); + for (i = 0; clk_ids[i]; i++) + num_clks++; + + adsp->num_clks = num_clks; adsp->clks = devm_kcalloc(adsp->dev, adsp->num_clks, sizeof(*adsp->clks), GFP_KERNEL); if (!adsp->clks) return -ENOMEM; for (i = 0; i < adsp->num_clks; i++) - adsp->clks[i].id = adsp_clk_id[i]; + adsp->clks[i].id = clk_ids[i]; return devm_clk_bulk_get(adsp->dev, adsp->num_clks, adsp->clks); } static int adsp_init_reset(struct qcom_adsp *adsp) { - adsp->pdc_sync_reset = devm_reset_control_get_exclusive(adsp->dev, + adsp->pdc_sync_reset = devm_reset_control_get_optional_exclusive(adsp->dev, "pdc_sync"); if (IS_ERR(adsp->pdc_sync_reset)) { dev_err(adsp->dev, "failed to acquire pdc_sync reset\n"); return PTR_ERR(adsp->pdc_sync_reset); } - adsp->cc_lpass_restart = devm_reset_control_get_exclusive(adsp->dev, - "cc_lpass"); - if (IS_ERR(adsp->cc_lpass_restart)) { - dev_err(adsp->dev, "failed to acquire cc_lpass restart\n"); - return PTR_ERR(adsp->cc_lpass_restart); + adsp->restart = devm_reset_control_get_optional_exclusive(adsp->dev, "restart"); + + /* Fall back to the old "cc_lpass" if "restart" is absent */ + if (!adsp->restart) + adsp->restart = devm_reset_control_get_exclusive(adsp->dev, "cc_lpass"); + + if (IS_ERR(adsp->restart)) { + dev_err(adsp->dev, "failed to acquire restart\n"); + return PTR_ERR(adsp->restart); } return 0; @@ -415,7 +433,7 @@ static int adsp_probe(struct platform_device *pdev) if (ret) goto free_rproc; - ret = adsp_init_clock(adsp); + ret = adsp_init_clock(adsp, desc->clk_ids); if (ret) goto free_rproc; @@ -479,9 +497,28 @@ static const struct adsp_pil_data adsp_resource_init = { .ssr_name = "lpass", .sysmon_name = "adsp", .ssctl_id = 0x14, + .clk_ids = (const char*[]) { + "sway_cbcr", "lpass_ahbs_aon_cbcr", "lpass_ahbm_aon_cbcr", + "qdsp6ss_xo", "qdsp6ss_sleep", "qdsp6ss_core", NULL + }, + .num_clks = 7, +}; + +static const struct adsp_pil_data cdsp_resource_init = { + .crash_reason_smem = 601, + .firmware_name = "cdsp.mdt", + .ssr_name = "cdsp", + .sysmon_name = "cdsp", + .ssctl_id = 0x17, + .clk_ids = (const char*[]) { + "sway", "tbu", "bimc", "ahb_aon", "q6ss_slave", "q6ss_master", + "q6_axim", NULL + }, + .num_clks = 7, }; static const struct of_device_id adsp_of_match[] = { + { .compatible = "qcom,qcs404-cdsp-pil", .data = &cdsp_resource_init }, { .compatible = "qcom,sdm845-adsp-pil", .data = &adsp_resource_init }, { }, }; diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 981581bcdd56..8fcf9d28dd73 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -659,23 +659,29 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) { unsigned long dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS; dma_addr_t phys; + void *metadata; int mdata_perm; int xferop_ret; + size_t size; void *ptr; int ret; - ptr = dma_alloc_attrs(qproc->dev, fw->size, &phys, GFP_KERNEL, dma_attrs); + metadata = qcom_mdt_read_metadata(fw, &size); + if (IS_ERR(metadata)) + return PTR_ERR(metadata); + + ptr = dma_alloc_attrs(qproc->dev, size, &phys, GFP_KERNEL, dma_attrs); if (!ptr) { + kfree(metadata); dev_err(qproc->dev, "failed to allocate mdt buffer\n"); return -ENOMEM; } - memcpy(ptr, fw->data, fw->size); + memcpy(ptr, metadata, size); /* Hypervisor mapping to access metadata by modem */ mdata_perm = BIT(QCOM_SCM_VMID_HLOS); - ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, - true, phys, fw->size); + ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, true, phys, size); if (ret) { dev_err(qproc->dev, "assigning Q6 access to metadata failed: %d\n", ret); @@ -693,14 +699,14 @@ static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw) dev_err(qproc->dev, "MPSS header authentication failed: %d\n", ret); /* Metadata authentication done, remove modem access */ - xferop_ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, - false, phys, fw->size); + xferop_ret = q6v5_xfer_mem_ownership(qproc, &mdata_perm, false, phys, size); if (xferop_ret) dev_warn(qproc->dev, "mdt buffer not reclaimed system may become unstable\n"); free_dma_attrs: - dma_free_attrs(qproc->dev, fw->size, ptr, phys, dma_attrs); + dma_free_attrs(qproc->dev, size, ptr, phys, dma_attrs); + kfree(metadata); return ret < 0 ? ret : 0; } @@ -981,7 +987,18 @@ static int q6v5_mpss_load(struct q6v5 *qproc) ptr = qproc->mpss_region + offset; - if (phdr->p_filesz) { + if (phdr->p_filesz && phdr->p_offset < fw->size) { + /* Firmware is large enough to be non-split */ + if (phdr->p_offset + phdr->p_filesz > fw->size) { + dev_err(qproc->dev, + "failed to load segment %d from truncated file %s\n", + i, fw_name); + ret = -EINVAL; + goto release_firmware; + } + + memcpy(ptr, fw->data + phdr->p_offset, phdr->p_filesz); + } else if (phdr->p_filesz) { /* Replace "xxx.xxx" with "xxx.bxx" */ sprintf(fw_name + fw_name_len - 3, "b%02d", i); ret = request_firmware(&seg_fw, fw_name, qproc->dev); diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 8b5363223eaa..3c5fbbbfb0f1 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -512,6 +512,7 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc, /* Initialise vdev subdevice */ snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index); rvdev->dev.parent = rproc->dev.parent; + rvdev->dev.dma_pfn_offset = rproc->dev.parent->dma_pfn_offset; rvdev->dev.release = rproc_rvdev_release; dev_set_name(&rvdev->dev, "%s#%s", dev_name(rvdev->dev.parent), name); dev_set_drvdata(&rvdev->dev, rvdev); @@ -1058,6 +1059,20 @@ static int rproc_handle_resources(struct rproc *rproc, dev_dbg(dev, "rsc: type %d\n", hdr->type); + if (hdr->type >= RSC_VENDOR_START && + hdr->type <= RSC_VENDOR_END) { + ret = rproc_handle_rsc(rproc, hdr->type, rsc, + offset + sizeof(*hdr), avail); + if (ret == RSC_HANDLED) + continue; + else if (ret < 0) + break; + + dev_warn(dev, "unsupported vendor resource %d\n", + hdr->type); + continue; + } + if (hdr->type >= RSC_LAST) { dev_warn(dev, "unsupported resource %d\n", hdr->type); continue; diff --git a/drivers/remoteproc/remoteproc_elf_loader.c b/drivers/remoteproc/remoteproc_elf_loader.c index 215a4400f21e..606aae166eba 100644 --- a/drivers/remoteproc/remoteproc_elf_loader.c +++ b/drivers/remoteproc/remoteproc_elf_loader.c @@ -247,8 +247,7 @@ find_table(struct device *dev, struct elf32_hdr *ehdr, size_t fw_size) } /* make sure the offsets array isn't truncated */ - if (table->num * sizeof(table->offset[0]) + - sizeof(struct resource_table) > size) { + if (struct_size(table, offset, table->num) > size) { dev_err(dev, "resource table incomplete\n"); return NULL; } diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h index a87750903785..493ef9262411 100644 --- a/drivers/remoteproc/remoteproc_internal.h +++ b/drivers/remoteproc/remoteproc_internal.h @@ -99,6 +99,17 @@ static inline int rproc_parse_fw(struct rproc *rproc, const struct firmware *fw) } static inline +int rproc_handle_rsc(struct rproc *rproc, u32 rsc_type, void *rsc, int offset, + int avail) +{ + if (rproc->ops->handle_rsc) + return rproc->ops->handle_rsc(rproc, rsc_type, rsc, offset, + avail); + + return RSC_IGNORED; +} + +static inline struct resource_table *rproc_find_loaded_rsc_table(struct rproc *rproc, const struct firmware *fw) { diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c new file mode 100644 index 000000000000..e2da7198b65f --- /dev/null +++ b/drivers/remoteproc/stm32_rproc.c @@ -0,0 +1,628 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) STMicroelectronics 2018 - All Rights Reserved + * Authors: Ludovic Barre <ludovic.barre@st.com> for STMicroelectronics. + * Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics. + */ + +#include <linux/arm-smccc.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/mailbox_client.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_reserved_mem.h> +#include <linux/regmap.h> +#include <linux/remoteproc.h> +#include <linux/reset.h> + +#include "remoteproc_internal.h" + +#define HOLD_BOOT 0 +#define RELEASE_BOOT 1 + +#define MBOX_NB_VQ 2 +#define MBOX_NB_MBX 3 + +#define STM32_SMC_RCC 0x82001000 +#define STM32_SMC_REG_WRITE 0x1 + +#define STM32_MBX_VQ0 "vq0" +#define STM32_MBX_VQ1 "vq1" +#define STM32_MBX_SHUTDOWN "shutdown" + +struct stm32_syscon { + struct regmap *map; + u32 reg; + u32 mask; +}; + +struct stm32_rproc_mem { + char name[20]; + void __iomem *cpu_addr; + phys_addr_t bus_addr; + u32 dev_addr; + size_t size; +}; + +struct stm32_rproc_mem_ranges { + u32 dev_addr; + u32 bus_addr; + u32 size; +}; + +struct stm32_mbox { + const unsigned char name[10]; + struct mbox_chan *chan; + struct mbox_client client; + int vq_id; +}; + +struct stm32_rproc { + struct reset_control *rst; + struct stm32_syscon hold_boot; + struct stm32_syscon pdds; + u32 nb_rmems; + struct stm32_rproc_mem *rmems; + struct stm32_mbox mb[MBOX_NB_MBX]; + bool secured_soc; +}; + +static int stm32_rproc_pa_to_da(struct rproc *rproc, phys_addr_t pa, u64 *da) +{ + unsigned int i; + struct stm32_rproc *ddata = rproc->priv; + struct stm32_rproc_mem *p_mem; + + for (i = 0; i < ddata->nb_rmems; i++) { + p_mem = &ddata->rmems[i]; + + if (pa < p_mem->bus_addr || + pa >= p_mem->bus_addr + p_mem->size) + continue; + *da = pa - p_mem->bus_addr + p_mem->dev_addr; + dev_dbg(rproc->dev.parent, "pa %pa to da %llx\n", &pa, *da); + return 0; + } + + return -EINVAL; +} + +static int stm32_rproc_mem_alloc(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + struct device *dev = rproc->dev.parent; + void *va; + + dev_dbg(dev, "map memory: %pa+%x\n", &mem->dma, mem->len); + va = ioremap_wc(mem->dma, mem->len); + if (IS_ERR_OR_NULL(va)) { + dev_err(dev, "Unable to map memory region: %pa+%x\n", + &mem->dma, mem->len); + return -ENOMEM; + } + + /* Update memory entry va */ + mem->va = va; + + return 0; +} + +static int stm32_rproc_mem_release(struct rproc *rproc, + struct rproc_mem_entry *mem) +{ + dev_dbg(rproc->dev.parent, "unmap memory: %pa\n", &mem->dma); + iounmap(mem->va); + + return 0; +} + +static int stm32_rproc_of_memory_translations(struct rproc *rproc) +{ + struct device *parent, *dev = rproc->dev.parent; + struct stm32_rproc *ddata = rproc->priv; + struct device_node *np; + struct stm32_rproc_mem *p_mems; + struct stm32_rproc_mem_ranges *mem_range; + int cnt, array_size, i, ret = 0; + + parent = dev->parent; + np = parent->of_node; + + cnt = of_property_count_elems_of_size(np, "dma-ranges", + sizeof(*mem_range)); + if (cnt <= 0) { + dev_err(dev, "%s: dma-ranges property not defined\n", __func__); + return -EINVAL; + } + + p_mems = devm_kcalloc(dev, cnt, sizeof(*p_mems), GFP_KERNEL); + if (!p_mems) + return -ENOMEM; + mem_range = kcalloc(cnt, sizeof(*mem_range), GFP_KERNEL); + if (!mem_range) + return -ENOMEM; + + array_size = cnt * sizeof(struct stm32_rproc_mem_ranges) / sizeof(u32); + + ret = of_property_read_u32_array(np, "dma-ranges", + (u32 *)mem_range, array_size); + if (ret) { + dev_err(dev, "error while get dma-ranges property: %x\n", ret); + goto free_mem; + } + + for (i = 0; i < cnt; i++) { + p_mems[i].bus_addr = mem_range[i].bus_addr; + p_mems[i].dev_addr = mem_range[i].dev_addr; + p_mems[i].size = mem_range[i].size; + + dev_dbg(dev, "memory range[%i]: da %#x, pa %pa, size %#zx:\n", + i, p_mems[i].dev_addr, &p_mems[i].bus_addr, + p_mems[i].size); + } + + ddata->rmems = p_mems; + ddata->nb_rmems = cnt; + +free_mem: + kfree(mem_range); + return ret; +} + +static int stm32_rproc_mbox_idx(struct rproc *rproc, const unsigned char *name) +{ + struct stm32_rproc *ddata = rproc->priv; + int i; + + for (i = 0; i < ARRAY_SIZE(ddata->mb); i++) { + if (!strncmp(ddata->mb[i].name, name, strlen(name))) + return i; + } + dev_err(&rproc->dev, "mailbox %s not found\n", name); + + return -EINVAL; +} + +static int stm32_rproc_elf_load_rsc_table(struct rproc *rproc, + const struct firmware *fw) +{ + if (rproc_elf_load_rsc_table(rproc, fw)) + dev_warn(&rproc->dev, "no resource table found for this firmware\n"); + + return 0; +} + +static int stm32_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw) +{ + struct device *dev = rproc->dev.parent; + struct device_node *np = dev->of_node; + struct of_phandle_iterator it; + struct rproc_mem_entry *mem; + struct reserved_mem *rmem; + u64 da; + int index = 0; + + /* Register associated reserved memory regions */ + of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); + while (of_phandle_iterator_next(&it) == 0) { + rmem = of_reserved_mem_lookup(it.node); + if (!rmem) { + dev_err(dev, "unable to acquire memory-region\n"); + return -EINVAL; + } + + if (stm32_rproc_pa_to_da(rproc, rmem->base, &da) < 0) { + dev_err(dev, "memory region not valid %pa\n", + &rmem->base); + return -EINVAL; + } + + /* No need to map vdev buffer */ + if (strcmp(it.node->name, "vdev0buffer")) { + /* Register memory region */ + mem = rproc_mem_entry_init(dev, NULL, + (dma_addr_t)rmem->base, + rmem->size, da, + stm32_rproc_mem_alloc, + stm32_rproc_mem_release, + it.node->name); + + if (mem) + rproc_coredump_add_segment(rproc, da, + rmem->size); + } else { + /* Register reserved memory for vdev buffer alloc */ + mem = rproc_of_resm_mem_entry_init(dev, index, + rmem->size, + rmem->base, + it.node->name); + } + + if (!mem) + return -ENOMEM; + + rproc_add_carveout(rproc, mem); + index++; + } + + return stm32_rproc_elf_load_rsc_table(rproc, fw); +} + +static irqreturn_t stm32_rproc_wdg(int irq, void *data) +{ + struct rproc *rproc = data; + + rproc_report_crash(rproc, RPROC_WATCHDOG); + + return IRQ_HANDLED; +} + +static void stm32_rproc_mb_callback(struct mbox_client *cl, void *data) +{ + struct rproc *rproc = dev_get_drvdata(cl->dev); + struct stm32_mbox *mb = container_of(cl, struct stm32_mbox, client); + + if (rproc_vq_interrupt(rproc, mb->vq_id) == IRQ_NONE) + dev_dbg(&rproc->dev, "no message found in vq%d\n", mb->vq_id); +} + +static void stm32_rproc_free_mbox(struct rproc *rproc) +{ + struct stm32_rproc *ddata = rproc->priv; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ddata->mb); i++) { + if (ddata->mb[i].chan) + mbox_free_channel(ddata->mb[i].chan); + ddata->mb[i].chan = NULL; + } +} + +static const struct stm32_mbox stm32_rproc_mbox[MBOX_NB_MBX] = { + { + .name = STM32_MBX_VQ0, + .vq_id = 0, + .client = { + .rx_callback = stm32_rproc_mb_callback, + .tx_block = false, + }, + }, + { + .name = STM32_MBX_VQ1, + .vq_id = 1, + .client = { + .rx_callback = stm32_rproc_mb_callback, + .tx_block = false, + }, + }, + { + .name = STM32_MBX_SHUTDOWN, + .vq_id = -1, + .client = { + .tx_block = true, + .tx_done = NULL, + .tx_tout = 500, /* 500 ms time out */ + }, + } +}; + +static void stm32_rproc_request_mbox(struct rproc *rproc) +{ + struct stm32_rproc *ddata = rproc->priv; + struct device *dev = &rproc->dev; + unsigned int i; + const unsigned char *name; + struct mbox_client *cl; + + /* Initialise mailbox structure table */ + memcpy(ddata->mb, stm32_rproc_mbox, sizeof(stm32_rproc_mbox)); + + for (i = 0; i < MBOX_NB_MBX; i++) { + name = ddata->mb[i].name; + + cl = &ddata->mb[i].client; + cl->dev = dev->parent; + + ddata->mb[i].chan = mbox_request_channel_byname(cl, name); + if (IS_ERR(ddata->mb[i].chan)) { + dev_warn(dev, "cannot get %s mbox\n", name); + ddata->mb[i].chan = NULL; + } + } +} + +static int stm32_rproc_set_hold_boot(struct rproc *rproc, bool hold) +{ + struct stm32_rproc *ddata = rproc->priv; + struct stm32_syscon hold_boot = ddata->hold_boot; + struct arm_smccc_res smc_res; + int val, err; + + val = hold ? HOLD_BOOT : RELEASE_BOOT; + + if (IS_ENABLED(CONFIG_HAVE_ARM_SMCCC) && ddata->secured_soc) { + arm_smccc_smc(STM32_SMC_RCC, STM32_SMC_REG_WRITE, + hold_boot.reg, val, 0, 0, 0, 0, &smc_res); + err = smc_res.a0; + } else { + err = regmap_update_bits(hold_boot.map, hold_boot.reg, + hold_boot.mask, val); + } + + if (err) + dev_err(&rproc->dev, "failed to set hold boot\n"); + + return err; +} + +static void stm32_rproc_add_coredump_trace(struct rproc *rproc) +{ + struct rproc_debug_trace *trace; + struct rproc_dump_segment *segment; + bool already_added; + + list_for_each_entry(trace, &rproc->traces, node) { + already_added = false; + + list_for_each_entry(segment, &rproc->dump_segments, node) { + if (segment->da == trace->trace_mem.da) { + already_added = true; + break; + } + } + + if (!already_added) + rproc_coredump_add_segment(rproc, trace->trace_mem.da, + trace->trace_mem.len); + } +} + +static int stm32_rproc_start(struct rproc *rproc) +{ + int err; + + stm32_rproc_add_coredump_trace(rproc); + + err = stm32_rproc_set_hold_boot(rproc, false); + if (err) + return err; + + return stm32_rproc_set_hold_boot(rproc, true); +} + +static int stm32_rproc_stop(struct rproc *rproc) +{ + struct stm32_rproc *ddata = rproc->priv; + int err, dummy_data, idx; + + /* request shutdown of the remote processor */ + if (rproc->state != RPROC_OFFLINE) { + idx = stm32_rproc_mbox_idx(rproc, STM32_MBX_SHUTDOWN); + if (idx >= 0 && ddata->mb[idx].chan) { + /* a dummy data is sent to allow to block on transmit */ + err = mbox_send_message(ddata->mb[idx].chan, + &dummy_data); + if (err < 0) + dev_warn(&rproc->dev, "warning: remote FW shutdown without ack\n"); + } + } + + err = stm32_rproc_set_hold_boot(rproc, true); + if (err) + return err; + + err = reset_control_assert(ddata->rst); + if (err) { + dev_err(&rproc->dev, "failed to assert the reset\n"); + return err; + } + + /* to allow platform Standby power mode, set remote proc Deep Sleep */ + if (ddata->pdds.map) { + err = regmap_update_bits(ddata->pdds.map, ddata->pdds.reg, + ddata->pdds.mask, 1); + if (err) { + dev_err(&rproc->dev, "failed to set pdds\n"); + return err; + } + } + + return 0; +} + +static void stm32_rproc_kick(struct rproc *rproc, int vqid) +{ + struct stm32_rproc *ddata = rproc->priv; + unsigned int i; + int err; + + if (WARN_ON(vqid >= MBOX_NB_VQ)) + return; + + for (i = 0; i < MBOX_NB_MBX; i++) { + if (vqid != ddata->mb[i].vq_id) + continue; + if (!ddata->mb[i].chan) + return; + err = mbox_send_message(ddata->mb[i].chan, (void *)(long)vqid); + if (err < 0) + dev_err(&rproc->dev, "%s: failed (%s, err:%d)\n", + __func__, ddata->mb[i].name, err); + return; + } +} + +static struct rproc_ops st_rproc_ops = { + .start = stm32_rproc_start, + .stop = stm32_rproc_stop, + .kick = stm32_rproc_kick, + .load = rproc_elf_load_segments, + .parse_fw = stm32_rproc_parse_fw, + .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, + .sanity_check = rproc_elf_sanity_check, + .get_boot_addr = rproc_elf_get_boot_addr, +}; + +static const struct of_device_id stm32_rproc_match[] = { + { .compatible = "st,stm32mp1-m4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, stm32_rproc_match); + +static int stm32_rproc_get_syscon(struct device_node *np, const char *prop, + struct stm32_syscon *syscon) +{ + int err = 0; + + syscon->map = syscon_regmap_lookup_by_phandle(np, prop); + if (IS_ERR(syscon->map)) { + err = PTR_ERR(syscon->map); + syscon->map = NULL; + goto out; + } + + err = of_property_read_u32_index(np, prop, 1, &syscon->reg); + if (err) + goto out; + + err = of_property_read_u32_index(np, prop, 2, &syscon->mask); + +out: + return err; +} + +static int stm32_rproc_parse_dt(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct rproc *rproc = platform_get_drvdata(pdev); + struct stm32_rproc *ddata = rproc->priv; + struct stm32_syscon tz; + unsigned int tzen; + int err, irq; + + irq = platform_get_irq(pdev, 0); + if (irq > 0) { + err = devm_request_irq(dev, irq, stm32_rproc_wdg, 0, + dev_name(dev), rproc); + if (err) { + dev_err(dev, "failed to request wdg irq\n"); + return err; + } + + dev_info(dev, "wdg irq registered\n"); + } + + ddata->rst = devm_reset_control_get_by_index(dev, 0); + if (IS_ERR(ddata->rst)) { + dev_err(dev, "failed to get mcu reset\n"); + return PTR_ERR(ddata->rst); + } + + /* + * if platform is secured the hold boot bit must be written by + * smc call and read normally. + * if not secure the hold boot bit could be read/write normally + */ + err = stm32_rproc_get_syscon(np, "st,syscfg-tz", &tz); + if (err) { + dev_err(dev, "failed to get tz syscfg\n"); + return err; + } + + err = regmap_read(tz.map, tz.reg, &tzen); + if (err) { + dev_err(&rproc->dev, "failed to read tzen\n"); + return err; + } + ddata->secured_soc = tzen & tz.mask; + + err = stm32_rproc_get_syscon(np, "st,syscfg-holdboot", + &ddata->hold_boot); + if (err) { + dev_err(dev, "failed to get hold boot\n"); + return err; + } + + err = stm32_rproc_get_syscon(np, "st,syscfg-pdds", &ddata->pdds); + if (err) + dev_warn(dev, "failed to get pdds\n"); + + rproc->auto_boot = of_property_read_bool(np, "st,auto-boot"); + + return stm32_rproc_of_memory_translations(rproc); +} + +static int stm32_rproc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct stm32_rproc *ddata; + struct device_node *np = dev->of_node; + struct rproc *rproc; + int ret; + + ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + rproc = rproc_alloc(dev, np->name, &st_rproc_ops, NULL, sizeof(*ddata)); + if (!rproc) + return -ENOMEM; + + rproc->has_iommu = false; + ddata = rproc->priv; + + platform_set_drvdata(pdev, rproc); + + ret = stm32_rproc_parse_dt(pdev); + if (ret) + goto free_rproc; + + stm32_rproc_request_mbox(rproc); + + ret = rproc_add(rproc); + if (ret) + goto free_mb; + + return 0; + +free_mb: + stm32_rproc_free_mbox(rproc); +free_rproc: + rproc_free(rproc); + return ret; +} + +static int stm32_rproc_remove(struct platform_device *pdev) +{ + struct rproc *rproc = platform_get_drvdata(pdev); + + if (atomic_read(&rproc->power) > 0) + rproc_shutdown(rproc); + + rproc_del(rproc); + stm32_rproc_free_mbox(rproc); + rproc_free(rproc); + + return 0; +} + +static struct platform_driver stm32_rproc_driver = { + .probe = stm32_rproc_probe, + .remove = stm32_rproc_remove, + .driver = { + .name = "stm32-rproc", + .of_match_table = of_match_ptr(stm32_rproc_match), + }, +}; +module_platform_driver(stm32_rproc_driver); + +MODULE_DESCRIPTION("STM32 Remote Processor Control Driver"); +MODULE_AUTHOR("Ludovic Barre <ludovic.barre@st.com>"); +MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); +MODULE_LICENSE("GPL v2"); + diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index 8122807db380..ea88fd4e2a6e 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -493,7 +493,8 @@ static int rpmsg_dev_remove(struct device *dev) if (rpdev->ops->announce_destroy) err = rpdev->ops->announce_destroy(rpdev); - rpdrv->remove(rpdev); + if (rpdrv->remove) + rpdrv->remove(rpdev); dev_pm_domain_detach(dev, true); diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 3254a30ebb1b..e72f65b61176 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -562,7 +562,7 @@ config RTC_DRV_TPS6586X config RTC_DRV_TPS65910 tristate "TI TPS65910 RTC driver" - depends on RTC_CLASS && MFD_TPS65910 + depends on MFD_TPS65910 help If you say yes here you get support for the RTC on the TPS65910 chips. @@ -820,6 +820,7 @@ config RTC_DRV_MAX6902 config RTC_DRV_PCF2123 tristate "NXP PCF2123" + select REGMAP_SPI help If you say yes here you get support for the NXP PCF2123 RTC chip. diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c index 4124f4dd376b..72b7ddc43116 100644 --- a/drivers/rtc/interface.c +++ b/drivers/rtc/interface.c @@ -633,7 +633,7 @@ enum hrtimer_restart rtc_pie_update_irq(struct hrtimer *timer) { struct rtc_device *rtc; ktime_t period; - int count; + u64 count; rtc = container_of(timer, struct rtc_device, pie_timer); diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c index 93d338e7732b..1f7e8aefc1eb 100644 --- a/drivers/rtc/rtc-ds1307.c +++ b/drivers/rtc/rtc-ds1307.c @@ -222,6 +222,45 @@ static int ds1307_get_time(struct device *dev, struct rtc_time *t) return -EINVAL; } + tmp = regs[DS1307_REG_SECS]; + switch (ds1307->type) { + case ds_1307: + case m41t0: + case m41t00: + case m41t11: + if (tmp & DS1307_BIT_CH) + return -EINVAL; + break; + case ds_1308: + case ds_1338: + if (tmp & DS1307_BIT_CH) + return -EINVAL; + + ret = regmap_read(ds1307->regmap, DS1307_REG_CONTROL, &tmp); + if (ret) + return ret; + if (tmp & DS1338_BIT_OSF) + return -EINVAL; + break; + case ds_1340: + if (tmp & DS1340_BIT_nEOSC) + return -EINVAL; + + ret = regmap_read(ds1307->regmap, DS1340_REG_FLAG, &tmp); + if (ret) + return ret; + if (tmp & DS1340_BIT_OSF) + return -EINVAL; + break; + case mcp794xx: + if (!(tmp & MCP794XX_BIT_ST)) + return -EINVAL; + + break; + default: + break; + } + t->tm_sec = bcd2bin(regs[DS1307_REG_SECS] & 0x7f); t->tm_min = bcd2bin(regs[DS1307_REG_MIN] & 0x7f); tmp = regs[DS1307_REG_HOUR] & 0x3f; @@ -286,7 +325,17 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t) if (t->tm_year > 199 && chip->century_bit) regs[chip->century_reg] |= chip->century_bit; - if (ds1307->type == mcp794xx) { + switch (ds1307->type) { + case ds_1308: + case ds_1338: + regmap_update_bits(ds1307->regmap, DS1307_REG_CONTROL, + DS1338_BIT_OSF, 0); + break; + case ds_1340: + regmap_update_bits(ds1307->regmap, DS1340_REG_FLAG, + DS1340_BIT_OSF, 0); + break; + case mcp794xx: /* * these bits were cleared when preparing the date/time * values and need to be set again before writing the @@ -294,6 +343,9 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t) */ regs[DS1307_REG_SECS] |= MCP794XX_BIT_ST; regs[DS1307_REG_WDAY] |= MCP794XX_BIT_VBATEN; + break; + default: + break; } dev_dbg(dev, "%s: %7ph\n", "write", regs); @@ -1702,7 +1754,6 @@ static int ds1307_probe(struct i2c_client *client, break; } -read_rtc: /* read RTC registers */ err = regmap_bulk_read(ds1307->regmap, chip->offset, regs, sizeof(regs)); @@ -1711,75 +1762,11 @@ read_rtc: goto exit; } - /* - * minimal sanity checking; some chips (like DS1340) don't - * specify the extra bits as must-be-zero, but there are - * still a few values that are clearly out-of-range. - */ - tmp = regs[DS1307_REG_SECS]; - switch (ds1307->type) { - case ds_1307: - case m41t0: - case m41t00: - case m41t11: - /* clock halted? turn it on, so clock can tick. */ - if (tmp & DS1307_BIT_CH) { - regmap_write(ds1307->regmap, DS1307_REG_SECS, 0); - dev_warn(ds1307->dev, "SET TIME!\n"); - goto read_rtc; - } - break; - case ds_1308: - case ds_1338: - /* clock halted? turn it on, so clock can tick. */ - if (tmp & DS1307_BIT_CH) - regmap_write(ds1307->regmap, DS1307_REG_SECS, 0); - - /* oscillator fault? clear flag, and warn */ - if (regs[DS1307_REG_CONTROL] & DS1338_BIT_OSF) { - regmap_write(ds1307->regmap, DS1307_REG_CONTROL, - regs[DS1307_REG_CONTROL] & - ~DS1338_BIT_OSF); - dev_warn(ds1307->dev, "SET TIME!\n"); - goto read_rtc; - } - break; - case ds_1340: - /* clock halted? turn it on, so clock can tick. */ - if (tmp & DS1340_BIT_nEOSC) - regmap_write(ds1307->regmap, DS1307_REG_SECS, 0); - - err = regmap_read(ds1307->regmap, DS1340_REG_FLAG, &tmp); - if (err) { - dev_dbg(ds1307->dev, "read error %d\n", err); - goto exit; - } - - /* oscillator fault? clear flag, and warn */ - if (tmp & DS1340_BIT_OSF) { - regmap_write(ds1307->regmap, DS1340_REG_FLAG, 0); - dev_warn(ds1307->dev, "SET TIME!\n"); - } - break; - case mcp794xx: - /* make sure that the backup battery is enabled */ - if (!(regs[DS1307_REG_WDAY] & MCP794XX_BIT_VBATEN)) { - regmap_write(ds1307->regmap, DS1307_REG_WDAY, - regs[DS1307_REG_WDAY] | - MCP794XX_BIT_VBATEN); - } - - /* clock halted? turn it on, so clock can tick. */ - if (!(tmp & MCP794XX_BIT_ST)) { - regmap_write(ds1307->regmap, DS1307_REG_SECS, - MCP794XX_BIT_ST); - dev_warn(ds1307->dev, "SET TIME!\n"); - goto read_rtc; - } - - break; - default: - break; + if (ds1307->type == mcp794xx && + !(regs[DS1307_REG_WDAY] & MCP794XX_BIT_VBATEN)) { + regmap_write(ds1307->regmap, DS1307_REG_WDAY, + regs[DS1307_REG_WDAY] | + MCP794XX_BIT_VBATEN); } tmp = regs[DS1307_REG_HOUR]; diff --git a/drivers/rtc/rtc-ds2404.c b/drivers/rtc/rtc-ds2404.c index 1e9f429ada64..9df0c44512b8 100644 --- a/drivers/rtc/rtc-ds2404.c +++ b/drivers/rtc/rtc-ds2404.c @@ -182,9 +182,10 @@ static void ds2404_enable_osc(struct device *dev) static int ds2404_read_time(struct device *dev, struct rtc_time *dt) { unsigned long time = 0; + __le32 hw_time = 0; - ds2404_read_memory(dev, 0x203, 4, (u8 *)&time); - time = le32_to_cpu(time); + ds2404_read_memory(dev, 0x203, 4, (u8 *)&hw_time); + time = le32_to_cpu(hw_time); rtc_time64_to_tm(time, dt); return 0; diff --git a/drivers/rtc/rtc-fm3130.c b/drivers/rtc/rtc-fm3130.c index 1caa21b82c7d..677ec2da13d8 100644 --- a/drivers/rtc/rtc-fm3130.c +++ b/drivers/rtc/rtc-fm3130.c @@ -104,8 +104,7 @@ static int fm3130_get_time(struct device *dev, struct rtc_time *t) fm3130_rtc_mode(dev, FM3130_MODE_READ); /* read the RTC date and time registers all at once */ - tmp = i2c_transfer(to_i2c_adapter(fm3130->client->dev.parent), - fm3130->msg, 2); + tmp = i2c_transfer(fm3130->client->adapter, fm3130->msg, 2); if (tmp != 2) { dev_err(dev, "%s error %d\n", "read", tmp); return -EIO; @@ -197,8 +196,7 @@ static int fm3130_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) } /* read the RTC alarm registers all at once */ - tmp = i2c_transfer(to_i2c_adapter(fm3130->client->dev.parent), - &fm3130->msg[2], 2); + tmp = i2c_transfer(fm3130->client->adapter, &fm3130->msg[2], 2); if (tmp != 2) { dev_err(dev, "%s error %d\n", "read", tmp); return -EIO; @@ -348,7 +346,7 @@ static int fm3130_probe(struct i2c_client *client, struct fm3130 *fm3130; int err = -ENODEV; int tmp; - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; if (!i2c_check_functionality(adapter, I2C_FUNC_I2C | I2C_FUNC_SMBUS_WRITE_BYTE_DATA)) diff --git a/drivers/rtc/rtc-imx-sc.c b/drivers/rtc/rtc-imx-sc.c index 19642bfd913a..c933045fe04b 100644 --- a/drivers/rtc/rtc-imx-sc.c +++ b/drivers/rtc/rtc-imx-sc.c @@ -3,6 +3,7 @@ * Copyright 2018 NXP. */ +#include <dt-bindings/firmware/imx/rsrc.h> #include <linux/arm-smccc.h> #include <linux/firmware/imx/sci.h> #include <linux/module.h> @@ -11,11 +12,15 @@ #include <linux/rtc.h> #define IMX_SC_TIMER_FUNC_GET_RTC_SEC1970 9 +#define IMX_SC_TIMER_FUNC_SET_RTC_ALARM 8 #define IMX_SC_TIMER_FUNC_SET_RTC_TIME 6 #define IMX_SIP_SRTC 0xC2000002 #define IMX_SIP_SRTC_SET_TIME 0x0 +#define SC_IRQ_GROUP_RTC 2 +#define SC_IRQ_RTC 1 + static struct imx_sc_ipc *rtc_ipc_handle; static struct rtc_device *imx_sc_rtc; @@ -24,6 +29,16 @@ struct imx_sc_msg_timer_get_rtc_time { u32 time; } __packed; +struct imx_sc_msg_timer_rtc_set_alarm { + struct imx_sc_rpc_msg hdr; + u16 year; + u8 mon; + u8 day; + u8 hour; + u8 min; + u8 sec; +} __packed; + static int imx_sc_rtc_read_time(struct device *dev, struct rtc_time *tm) { struct imx_sc_msg_timer_get_rtc_time msg; @@ -60,9 +75,77 @@ static int imx_sc_rtc_set_time(struct device *dev, struct rtc_time *tm) return res.a0; } +static int imx_sc_rtc_alarm_irq_enable(struct device *dev, unsigned int enable) +{ + return imx_scu_irq_group_enable(SC_IRQ_GROUP_RTC, SC_IRQ_RTC, enable); +} + +static int imx_sc_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + /* + * SCU firmware does NOT provide read alarm API, but .read_alarm + * callback is required by RTC framework to support alarm function, + * so just return here. + */ + return 0; +} + +static int imx_sc_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct imx_sc_msg_timer_rtc_set_alarm msg; + struct imx_sc_rpc_msg *hdr = &msg.hdr; + int ret; + struct rtc_time *alrm_tm = &alrm->time; + + hdr->ver = IMX_SC_RPC_VERSION; + hdr->svc = IMX_SC_RPC_SVC_TIMER; + hdr->func = IMX_SC_TIMER_FUNC_SET_RTC_ALARM; + hdr->size = 3; + + msg.year = alrm_tm->tm_year + 1900; + msg.mon = alrm_tm->tm_mon + 1; + msg.day = alrm_tm->tm_mday; + msg.hour = alrm_tm->tm_hour; + msg.min = alrm_tm->tm_min; + msg.sec = alrm_tm->tm_sec; + + ret = imx_scu_call_rpc(rtc_ipc_handle, &msg, true); + if (ret) { + dev_err(dev, "set rtc alarm failed, ret %d\n", ret); + return ret; + } + + ret = imx_sc_rtc_alarm_irq_enable(dev, alrm->enabled); + if (ret) { + dev_err(dev, "enable rtc alarm failed, ret %d\n", ret); + return ret; + } + + return 0; +} + static const struct rtc_class_ops imx_sc_rtc_ops = { .read_time = imx_sc_rtc_read_time, .set_time = imx_sc_rtc_set_time, + .read_alarm = imx_sc_rtc_read_alarm, + .set_alarm = imx_sc_rtc_set_alarm, + .alarm_irq_enable = imx_sc_rtc_alarm_irq_enable, +}; + +static int imx_sc_rtc_alarm_notify(struct notifier_block *nb, + unsigned long event, void *group) +{ + /* ignore non-rtc irq */ + if (!((event & SC_IRQ_RTC) && (*(u8 *)group == SC_IRQ_GROUP_RTC))) + return 0; + + rtc_update_irq(imx_sc_rtc, 1, RTC_IRQF | RTC_AF); + + return 0; +} + +static struct notifier_block imx_sc_rtc_alarm_sc_notifier = { + .notifier_call = imx_sc_rtc_alarm_notify, }; static int imx_sc_rtc_probe(struct platform_device *pdev) @@ -73,6 +156,8 @@ static int imx_sc_rtc_probe(struct platform_device *pdev) if (ret) return ret; + device_init_wakeup(&pdev->dev, true); + imx_sc_rtc = devm_rtc_allocate_device(&pdev->dev); if (IS_ERR(imx_sc_rtc)) return PTR_ERR(imx_sc_rtc); @@ -87,6 +172,8 @@ static int imx_sc_rtc_probe(struct platform_device *pdev) return ret; } + imx_scu_irq_register_notifier(&imx_sc_rtc_alarm_sc_notifier); + return 0; } diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c index 9fdc284c943b..5f46f85f814b 100644 --- a/drivers/rtc/rtc-m41t80.c +++ b/drivers/rtc/rtc-m41t80.c @@ -872,7 +872,7 @@ static struct notifier_block wdt_notifier = { static int m41t80_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; int rc = 0; struct rtc_time tm; struct m41t80_data *m41t80_data = NULL; diff --git a/drivers/rtc/rtc-pcf2123.c b/drivers/rtc/rtc-pcf2123.c index f431263e2d39..fb542a930bf0 100644 --- a/drivers/rtc/rtc-pcf2123.c +++ b/drivers/rtc/rtc-pcf2123.c @@ -40,7 +40,7 @@ #include <linux/rtc.h> #include <linux/spi/spi.h> #include <linux/module.h> -#include <linux/sysfs.h> +#include <linux/regmap.h> /* REGISTERS */ #define PCF2123_REG_CTRL1 (0x00) /* Control Register 1 */ @@ -95,6 +95,7 @@ #define OFFSET_SIGN_BIT 6 /* 2's complement sign bit */ #define OFFSET_COARSE BIT(7) /* Coarse mode offset */ #define OFFSET_STEP (2170) /* Offset step in parts per billion */ +#define OFFSET_MASK GENMASK(6, 0) /* Offset value */ /* READ/WRITE ADDRESS BITS */ #define PCF2123_WRITE BIT(4) @@ -103,120 +104,35 @@ static struct spi_driver pcf2123_driver; -struct pcf2123_sysfs_reg { - struct device_attribute attr; - char name[2]; -}; - struct pcf2123_plat_data { struct rtc_device *rtc; - struct pcf2123_sysfs_reg regs[16]; + struct regmap *map; }; -/* - * Causes a 30 nanosecond delay to ensure that the PCF2123 chip select - * is released properly after an SPI write. This function should be - * called after EVERY read/write call over SPI. - */ -static inline void pcf2123_delay_trec(void) -{ - ndelay(30); -} - -static int pcf2123_read(struct device *dev, u8 reg, u8 *rxbuf, size_t size) -{ - struct spi_device *spi = to_spi_device(dev); - int ret; - - reg |= PCF2123_READ; - ret = spi_write_then_read(spi, ®, 1, rxbuf, size); - pcf2123_delay_trec(); - - return ret; -} - -static int pcf2123_write(struct device *dev, u8 *txbuf, size_t size) -{ - struct spi_device *spi = to_spi_device(dev); - int ret; - - txbuf[0] |= PCF2123_WRITE; - ret = spi_write(spi, txbuf, size); - pcf2123_delay_trec(); - - return ret; -} - -static int pcf2123_write_reg(struct device *dev, u8 reg, u8 val) -{ - u8 txbuf[2]; - - txbuf[0] = reg; - txbuf[1] = val; - return pcf2123_write(dev, txbuf, sizeof(txbuf)); -} - -static ssize_t pcf2123_show(struct device *dev, struct device_attribute *attr, - char *buffer) -{ - struct pcf2123_sysfs_reg *r; - u8 rxbuf[1]; - unsigned long reg; - int ret; - - r = container_of(attr, struct pcf2123_sysfs_reg, attr); - - ret = kstrtoul(r->name, 16, ®); - if (ret) - return ret; - - ret = pcf2123_read(dev, reg, rxbuf, 1); - if (ret < 0) - return -EIO; - - return sprintf(buffer, "0x%x\n", rxbuf[0]); -} +static const struct regmap_config pcf2123_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .read_flag_mask = PCF2123_READ, + .write_flag_mask = PCF2123_WRITE, + .max_register = PCF2123_REG_CTDWN_TMR, +}; -static ssize_t pcf2123_store(struct device *dev, struct device_attribute *attr, - const char *buffer, size_t count) +static int pcf2123_read_offset(struct device *dev, long *offset) { - struct pcf2123_sysfs_reg *r; - unsigned long reg; - unsigned long val; - - int ret; + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); + int ret, val; + unsigned int reg; - r = container_of(attr, struct pcf2123_sysfs_reg, attr); - - ret = kstrtoul(r->name, 16, ®); + ret = regmap_read(pdata->map, PCF2123_REG_OFFSET, ®); if (ret) return ret; - ret = kstrtoul(buffer, 10, &val); - if (ret) - return ret; - - ret = pcf2123_write_reg(dev, reg, val); - if (ret < 0) - return -EIO; - return count; -} - -static int pcf2123_read_offset(struct device *dev, long *offset) -{ - int ret; - s8 reg; - - ret = pcf2123_read(dev, PCF2123_REG_OFFSET, ®, 1); - if (ret < 0) - return ret; + val = sign_extend32((reg & OFFSET_MASK), OFFSET_SIGN_BIT); if (reg & OFFSET_COARSE) - reg <<= 1; /* multiply by 2 and sign extend */ - else - reg = sign_extend32(reg, OFFSET_SIGN_BIT); + val *= 2; - *offset = ((long)reg) * OFFSET_STEP; + *offset = ((long)val) * OFFSET_STEP; return 0; } @@ -233,6 +149,7 @@ static int pcf2123_read_offset(struct device *dev, long *offset) */ static int pcf2123_set_offset(struct device *dev, long offset) { + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); s8 reg; if (offset > OFFSET_STEP * 127) @@ -240,7 +157,7 @@ static int pcf2123_set_offset(struct device *dev, long offset) else if (offset < OFFSET_STEP * -128) reg = -128; else - reg = (s8)((offset + (OFFSET_STEP >> 1)) / OFFSET_STEP); + reg = DIV_ROUND_CLOSEST(offset, OFFSET_STEP); /* choose fine offset only for odd values in the normal range */ if (reg & 1 && reg <= 63 && reg >= -64) { @@ -252,16 +169,18 @@ static int pcf2123_set_offset(struct device *dev, long offset) reg |= OFFSET_COARSE; } - return pcf2123_write_reg(dev, PCF2123_REG_OFFSET, reg); + return regmap_write(pdata->map, PCF2123_REG_OFFSET, (unsigned int)reg); } static int pcf2123_rtc_read_time(struct device *dev, struct rtc_time *tm) { + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); u8 rxbuf[7]; int ret; - ret = pcf2123_read(dev, PCF2123_REG_SC, rxbuf, sizeof(rxbuf)); - if (ret < 0) + ret = regmap_bulk_read(pdata->map, PCF2123_REG_SC, rxbuf, + sizeof(rxbuf)); + if (ret) return ret; if (rxbuf[0] & OSC_HAS_STOPPED) { @@ -279,82 +198,168 @@ static int pcf2123_rtc_read_time(struct device *dev, struct rtc_time *tm) if (tm->tm_year < 70) tm->tm_year += 100; /* assume we are in 1970...2069 */ - dev_dbg(dev, "%s: tm is secs=%d, mins=%d, hours=%d, " - "mday=%d, mon=%d, year=%d, wday=%d\n", - __func__, - tm->tm_sec, tm->tm_min, tm->tm_hour, - tm->tm_mday, tm->tm_mon, tm->tm_year, tm->tm_wday); + dev_dbg(dev, "%s: tm is %ptR\n", __func__, tm); return 0; } static int pcf2123_rtc_set_time(struct device *dev, struct rtc_time *tm) { - u8 txbuf[8]; + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); + u8 txbuf[7]; int ret; - dev_dbg(dev, "%s: tm is secs=%d, mins=%d, hours=%d, " - "mday=%d, mon=%d, year=%d, wday=%d\n", - __func__, - tm->tm_sec, tm->tm_min, tm->tm_hour, - tm->tm_mday, tm->tm_mon, tm->tm_year, tm->tm_wday); + dev_dbg(dev, "%s: tm is %ptR\n", __func__, tm); /* Stop the counter first */ - ret = pcf2123_write_reg(dev, PCF2123_REG_CTRL1, CTRL1_STOP); - if (ret < 0) + ret = regmap_write(pdata->map, PCF2123_REG_CTRL1, CTRL1_STOP); + if (ret) return ret; /* Set the new time */ - txbuf[0] = PCF2123_REG_SC; - txbuf[1] = bin2bcd(tm->tm_sec & 0x7F); - txbuf[2] = bin2bcd(tm->tm_min & 0x7F); - txbuf[3] = bin2bcd(tm->tm_hour & 0x3F); - txbuf[4] = bin2bcd(tm->tm_mday & 0x3F); - txbuf[5] = tm->tm_wday & 0x07; - txbuf[6] = bin2bcd((tm->tm_mon + 1) & 0x1F); /* rtc mn 1-12 */ - txbuf[7] = bin2bcd(tm->tm_year < 100 ? tm->tm_year : tm->tm_year - 100); - - ret = pcf2123_write(dev, txbuf, sizeof(txbuf)); - if (ret < 0) + txbuf[0] = bin2bcd(tm->tm_sec & 0x7F); + txbuf[1] = bin2bcd(tm->tm_min & 0x7F); + txbuf[2] = bin2bcd(tm->tm_hour & 0x3F); + txbuf[3] = bin2bcd(tm->tm_mday & 0x3F); + txbuf[4] = tm->tm_wday & 0x07; + txbuf[5] = bin2bcd((tm->tm_mon + 1) & 0x1F); /* rtc mn 1-12 */ + txbuf[6] = bin2bcd(tm->tm_year < 100 ? tm->tm_year : tm->tm_year - 100); + + ret = regmap_bulk_write(pdata->map, PCF2123_REG_SC, txbuf, + sizeof(txbuf)); + if (ret) return ret; /* Start the counter */ - ret = pcf2123_write_reg(dev, PCF2123_REG_CTRL1, CTRL1_CLEAR); - if (ret < 0) + ret = regmap_write(pdata->map, PCF2123_REG_CTRL1, CTRL1_CLEAR); + if (ret) + return ret; + + return 0; +} + +static int pcf2123_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm) +{ + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); + u8 rxbuf[4]; + int ret; + unsigned int val = 0; + + ret = regmap_bulk_read(pdata->map, PCF2123_REG_ALRM_MN, rxbuf, + sizeof(rxbuf)); + if (ret) + return ret; + + alm->time.tm_min = bcd2bin(rxbuf[0] & 0x7F); + alm->time.tm_hour = bcd2bin(rxbuf[1] & 0x3F); + alm->time.tm_mday = bcd2bin(rxbuf[2] & 0x3F); + alm->time.tm_wday = bcd2bin(rxbuf[3] & 0x07); + + dev_dbg(dev, "%s: alm is %ptR\n", __func__, &alm->time); + + ret = regmap_read(pdata->map, PCF2123_REG_CTRL2, &val); + if (ret) + return ret; + + alm->enabled = !!(val & CTRL2_AIE); + + return 0; +} + +static int pcf2123_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm) +{ + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); + u8 txbuf[4]; + int ret; + + dev_dbg(dev, "%s: alm is %ptR\n", __func__, &alm->time); + + /* Ensure alarm flag is clear */ + ret = regmap_update_bits(pdata->map, PCF2123_REG_CTRL2, CTRL2_AF, 0); + if (ret) return ret; + /* Disable alarm interrupt */ + ret = regmap_update_bits(pdata->map, PCF2123_REG_CTRL2, CTRL2_AIE, 0); + if (ret) + return ret; + + /* Set new alarm */ + txbuf[0] = bin2bcd(alm->time.tm_min & 0x7F); + txbuf[1] = bin2bcd(alm->time.tm_hour & 0x3F); + txbuf[2] = bin2bcd(alm->time.tm_mday & 0x3F); + txbuf[3] = bin2bcd(alm->time.tm_wday & 0x07); + + ret = regmap_bulk_write(pdata->map, PCF2123_REG_ALRM_MN, txbuf, + sizeof(txbuf)); + if (ret) + return ret; + + /* Enable alarm interrupt */ + if (alm->enabled) { + ret = regmap_update_bits(pdata->map, PCF2123_REG_CTRL2, + CTRL2_AIE, CTRL2_AIE); + if (ret) + return ret; + } + return 0; } +static irqreturn_t pcf2123_rtc_irq(int irq, void *dev) +{ + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); + struct mutex *lock = &pdata->rtc->ops_lock; + unsigned int val = 0; + int ret = IRQ_NONE; + + mutex_lock(lock); + regmap_read(pdata->map, PCF2123_REG_CTRL2, &val); + + /* Alarm? */ + if (val & CTRL2_AF) { + ret = IRQ_HANDLED; + + /* Clear alarm flag */ + regmap_update_bits(pdata->map, PCF2123_REG_CTRL2, CTRL2_AF, 0); + + rtc_update_irq(pdata->rtc, 1, RTC_IRQF | RTC_AF); + } + + mutex_unlock(lock); + + return ret; +} + static int pcf2123_reset(struct device *dev) { + struct pcf2123_plat_data *pdata = dev_get_platdata(dev); int ret; - u8 rxbuf[2]; + unsigned int val = 0; - ret = pcf2123_write_reg(dev, PCF2123_REG_CTRL1, CTRL1_SW_RESET); - if (ret < 0) + ret = regmap_write(pdata->map, PCF2123_REG_CTRL1, CTRL1_SW_RESET); + if (ret) return ret; /* Stop the counter */ dev_dbg(dev, "stopping RTC\n"); - ret = pcf2123_write_reg(dev, PCF2123_REG_CTRL1, CTRL1_STOP); - if (ret < 0) + ret = regmap_write(pdata->map, PCF2123_REG_CTRL1, CTRL1_STOP); + if (ret) return ret; /* See if the counter was actually stopped */ dev_dbg(dev, "checking for presence of RTC\n"); - ret = pcf2123_read(dev, PCF2123_REG_CTRL1, rxbuf, sizeof(rxbuf)); - if (ret < 0) + ret = regmap_read(pdata->map, PCF2123_REG_CTRL1, &val); + if (ret) return ret; - dev_dbg(dev, "received data from RTC (0x%02X 0x%02X)\n", - rxbuf[0], rxbuf[1]); - if (!(rxbuf[0] & CTRL1_STOP)) + dev_dbg(dev, "received data from RTC (0x%08X)\n", val); + if (!(val & CTRL1_STOP)) return -ENODEV; /* Start the counter */ - ret = pcf2123_write_reg(dev, PCF2123_REG_CTRL1, CTRL1_CLEAR); - if (ret < 0) + ret = regmap_write(pdata->map, PCF2123_REG_CTRL1, CTRL1_CLEAR); + if (ret) return ret; return 0; @@ -365,7 +370,8 @@ static const struct rtc_class_ops pcf2123_rtc_ops = { .set_time = pcf2123_rtc_set_time, .read_offset = pcf2123_read_offset, .set_offset = pcf2123_set_offset, - + .read_alarm = pcf2123_rtc_read_alarm, + .set_alarm = pcf2123_rtc_set_alarm, }; static int pcf2123_probe(struct spi_device *spi) @@ -373,7 +379,7 @@ static int pcf2123_probe(struct spi_device *spi) struct rtc_device *rtc; struct rtc_time tm; struct pcf2123_plat_data *pdata; - int ret, i; + int ret = 0; pdata = devm_kzalloc(&spi->dev, sizeof(struct pcf2123_plat_data), GFP_KERNEL); @@ -381,6 +387,13 @@ static int pcf2123_probe(struct spi_device *spi) return -ENOMEM; spi->dev.platform_data = pdata; + pdata->map = devm_regmap_init_spi(spi, &pcf2123_regmap_config); + + if (IS_ERR(pdata->map)) { + dev_err(&spi->dev, "regmap init failed.\n"); + goto kfree_exit; + } + ret = pcf2123_rtc_read_time(&spi->dev, &tm); if (ret < 0) { ret = pcf2123_reset(&spi->dev); @@ -405,47 +418,31 @@ static int pcf2123_probe(struct spi_device *spi) pdata->rtc = rtc; - for (i = 0; i < 16; i++) { - sysfs_attr_init(&pdata->regs[i].attr.attr); - sprintf(pdata->regs[i].name, "%1x", i); - pdata->regs[i].attr.attr.mode = S_IRUGO | S_IWUSR; - pdata->regs[i].attr.attr.name = pdata->regs[i].name; - pdata->regs[i].attr.show = pcf2123_show; - pdata->regs[i].attr.store = pcf2123_store; - ret = device_create_file(&spi->dev, &pdata->regs[i].attr); - if (ret) { - dev_err(&spi->dev, "Unable to create sysfs %s\n", - pdata->regs[i].name); - goto sysfs_exit; - } + /* Register alarm irq */ + if (spi->irq > 0) { + ret = devm_request_threaded_irq(&spi->dev, spi->irq, NULL, + pcf2123_rtc_irq, + IRQF_TRIGGER_LOW | IRQF_ONESHOT, + pcf2123_driver.driver.name, &spi->dev); + if (!ret) + device_init_wakeup(&spi->dev, true); + else + dev_err(&spi->dev, "could not request irq.\n"); } - return 0; + /* The PCF2123's alarm only has minute accuracy. Must add timer + * support to this driver to generate interrupts more than once + * per minute. + */ + pdata->rtc->uie_unsupported = 1; -sysfs_exit: - for (i--; i >= 0; i--) - device_remove_file(&spi->dev, &pdata->regs[i].attr); + return 0; kfree_exit: spi->dev.platform_data = NULL; return ret; } -static int pcf2123_remove(struct spi_device *spi) -{ - struct pcf2123_plat_data *pdata = dev_get_platdata(&spi->dev); - int i; - - if (pdata) { - for (i = 0; i < 16; i++) - if (pdata->regs[i].name[0]) - device_remove_file(&spi->dev, - &pdata->regs[i].attr); - } - - return 0; -} - #ifdef CONFIG_OF static const struct of_device_id pcf2123_dt_ids[] = { { .compatible = "nxp,rtc-pcf2123", }, @@ -461,7 +458,6 @@ static struct spi_driver pcf2123_driver = { .of_match_table = of_match_ptr(pcf2123_dt_ids), }, .probe = pcf2123_probe, - .remove = pcf2123_remove, }; module_spi_driver(pcf2123_driver); diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index c569dfe8c2ae..ac159d24286d 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -560,7 +560,6 @@ static int pcf8563_probe(struct i2c_client *client, struct pcf8563 *pcf8563; int err; unsigned char buf; - unsigned char alm_pending; dev_dbg(&client->dev, "%s\n", __func__); @@ -584,13 +583,13 @@ static int pcf8563_probe(struct i2c_client *client, return err; } - err = pcf8563_get_alarm_mode(client, NULL, &alm_pending); - if (err) { - dev_err(&client->dev, "%s: read error\n", __func__); + /* Clear flags and disable interrupts */ + buf = 0; + err = pcf8563_write_block_data(client, PCF8563_REG_ST2, 1, &buf); + if (err < 0) { + dev_err(&client->dev, "%s: write error\n", __func__); return err; } - if (alm_pending) - pcf8563_set_alarm_mode(client, 0); pcf8563->rtc = devm_rtc_device_register(&client->dev, pcf8563_driver.driver.name, @@ -602,7 +601,7 @@ static int pcf8563_probe(struct i2c_client *client, if (client->irq > 0) { err = devm_request_threaded_irq(&client->dev, client->irq, NULL, pcf8563_irq, - IRQF_SHARED|IRQF_ONESHOT|IRQF_TRIGGER_FALLING, + IRQF_SHARED | IRQF_ONESHOT | IRQF_TRIGGER_LOW, pcf8563_driver.driver.name, client); if (err) { dev_err(&client->dev, "unable to request IRQ %d\n", diff --git a/drivers/rtc/rtc-rv8803.c b/drivers/rtc/rtc-rv8803.c index 0b102c3cf5a4..fc5243400108 100644 --- a/drivers/rtc/rtc-rv8803.c +++ b/drivers/rtc/rtc-rv8803.c @@ -517,7 +517,7 @@ static int rx8900_trickle_charger_init(struct rv8803_data *rv8803) static int rv8803_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; struct rv8803_data *rv8803; int err, flags; struct nvmem_config nvmem_cfg = { diff --git a/drivers/rtc/rtc-rx8010.c b/drivers/rtc/rtc-rx8010.c index b15ad8e10938..8102469e27c0 100644 --- a/drivers/rtc/rtc-rx8010.c +++ b/drivers/rtc/rtc-rx8010.c @@ -433,7 +433,7 @@ static struct rtc_class_ops rx8010_rtc_ops = { static int rx8010_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; struct rx8010_data *rx8010; int err = 0; diff --git a/drivers/rtc/rtc-rx8025.c b/drivers/rtc/rtc-rx8025.c index cb082ad19471..b9bda10589e0 100644 --- a/drivers/rtc/rtc-rx8025.c +++ b/drivers/rtc/rtc-rx8025.c @@ -501,7 +501,7 @@ static void rx8025_sysfs_unregister(struct device *dev) static int rx8025_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent); + struct i2c_adapter *adapter = client->adapter; struct rx8025_data *rx8025; int err = 0; diff --git a/drivers/rtc/rtc-s35390a.c b/drivers/rtc/rtc-s35390a.c index 8c37acb4a007..84806ff763cf 100644 --- a/drivers/rtc/rtc-s35390a.c +++ b/drivers/rtc/rtc-s35390a.c @@ -32,21 +32,22 @@ #define S35390A_ALRM_BYTE_MINS 2 /* flags for STATUS1 */ -#define S35390A_FLAG_POC 0x01 -#define S35390A_FLAG_BLD 0x02 -#define S35390A_FLAG_INT2 0x04 -#define S35390A_FLAG_24H 0x40 -#define S35390A_FLAG_RESET 0x80 +#define S35390A_FLAG_POC BIT(0) +#define S35390A_FLAG_BLD BIT(1) +#define S35390A_FLAG_INT2 BIT(2) +#define S35390A_FLAG_24H BIT(6) +#define S35390A_FLAG_RESET BIT(7) /* flag for STATUS2 */ -#define S35390A_FLAG_TEST 0x01 - -#define S35390A_INT2_MODE_MASK 0xF0 +#define S35390A_FLAG_TEST BIT(0) +/* INT2 pin output mode */ +#define S35390A_INT2_MODE_MASK 0x0E #define S35390A_INT2_MODE_NOINTR 0x00 -#define S35390A_INT2_MODE_FREQ 0x10 -#define S35390A_INT2_MODE_ALARM 0x40 -#define S35390A_INT2_MODE_PMIN_EDG 0x20 +#define S35390A_INT2_MODE_ALARM BIT(1) /* INT2AE */ +#define S35390A_INT2_MODE_PMIN_EDG BIT(2) /* INT2ME */ +#define S35390A_INT2_MODE_FREQ BIT(3) /* INT2FE */ +#define S35390A_INT2_MODE_PMIN (BIT(3) | BIT(2)) /* INT2FE | INT2ME */ static const struct i2c_device_id s35390a_id[] = { { "s35390a", 0 }, @@ -284,6 +285,9 @@ static int s35390a_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm) alm->time.tm_min, alm->time.tm_hour, alm->time.tm_mday, alm->time.tm_mon, alm->time.tm_year, alm->time.tm_wday); + if (alm->time.tm_sec != 0) + dev_warn(&client->dev, "Alarms are only supported on a per minute basis!\n"); + /* disable interrupt (which deasserts the irq line) */ err = s35390a_set_reg(s35390a, S35390A_CMD_STATUS2, &sts, sizeof(sts)); if (err < 0) @@ -299,9 +303,6 @@ static int s35390a_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm) else sts = S35390A_INT2_MODE_NOINTR; - /* This chip expects the bits of each byte to be in reverse order */ - sts = bitrev8(sts); - /* set interupt mode*/ err = s35390a_set_reg(s35390a, S35390A_CMD_STATUS2, &sts, sizeof(sts)); if (err < 0) @@ -339,7 +340,7 @@ static int s35390a_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm) if (err < 0) return err; - if ((bitrev8(sts) & S35390A_INT2_MODE_MASK) != S35390A_INT2_MODE_ALARM) { + if ((sts & S35390A_INT2_MODE_MASK) != S35390A_INT2_MODE_ALARM) { /* * When the alarm isn't enabled, the register to configure * the alarm time isn't accessible. @@ -431,14 +432,14 @@ static int s35390a_probe(struct i2c_client *client, unsigned int i; struct s35390a *s35390a; char buf, status1; + struct device *dev = &client->dev; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { err = -ENODEV; goto exit; } - s35390a = devm_kzalloc(&client->dev, sizeof(struct s35390a), - GFP_KERNEL); + s35390a = devm_kzalloc(dev, sizeof(struct s35390a), GFP_KERNEL); if (!s35390a) { err = -ENOMEM; goto exit; @@ -452,8 +453,8 @@ static int s35390a_probe(struct i2c_client *client, s35390a->client[i] = i2c_new_dummy(client->adapter, client->addr + i); if (!s35390a->client[i]) { - dev_err(&client->dev, "Address %02x unavailable\n", - client->addr + i); + dev_err(dev, "Address %02x unavailable\n", + client->addr + i); err = -EBUSY; goto exit_dummy; } @@ -462,7 +463,7 @@ static int s35390a_probe(struct i2c_client *client, err_read = s35390a_read_status(s35390a, &status1); if (err_read < 0) { err = err_read; - dev_err(&client->dev, "error resetting chip\n"); + dev_err(dev, "error resetting chip\n"); goto exit_dummy; } @@ -476,28 +477,30 @@ static int s35390a_probe(struct i2c_client *client, buf = 0; err = s35390a_set_reg(s35390a, S35390A_CMD_STATUS2, &buf, 1); if (err < 0) { - dev_err(&client->dev, "error disabling alarm"); + dev_err(dev, "error disabling alarm"); goto exit_dummy; } } else { err = s35390a_disable_test_mode(s35390a); if (err < 0) { - dev_err(&client->dev, "error disabling test mode\n"); + dev_err(dev, "error disabling test mode\n"); goto exit_dummy; } } - device_set_wakeup_capable(&client->dev, 1); + device_set_wakeup_capable(dev, 1); - s35390a->rtc = devm_rtc_device_register(&client->dev, - s35390a_driver.driver.name, - &s35390a_rtc_ops, THIS_MODULE); + s35390a->rtc = devm_rtc_device_register(dev, s35390a_driver.driver.name, + &s35390a_rtc_ops, THIS_MODULE); if (IS_ERR(s35390a->rtc)) { err = PTR_ERR(s35390a->rtc); goto exit_dummy; } + /* supports per-minute alarms only, therefore set uie_unsupported */ + s35390a->rtc->uie_unsupported = 1; + if (status1 & S35390A_FLAG_INT2) rtc_update_irq(s35390a->rtc, 1, RTC_AF); diff --git a/drivers/rtc/rtc-st-lpc.c b/drivers/rtc/rtc-st-lpc.c index 5fe021821831..49474a31c66d 100644 --- a/drivers/rtc/rtc-st-lpc.c +++ b/drivers/rtc/rtc-st-lpc.c @@ -162,10 +162,6 @@ static int st_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *t) now_secs = rtc_tm_to_time64(&now); alarm_secs = rtc_tm_to_time64(&t->time); - /* Invalid alarm time */ - if (now_secs > alarm_secs) - return -EINVAL; - memcpy(&rtc->alarm, t, sizeof(struct rtc_wkalrm)); /* Now many secs to fire */ diff --git a/drivers/rtc/rtc-stm32.c b/drivers/rtc/rtc-stm32.c index 8e6c9b3bcc29..773a1990b93f 100644 --- a/drivers/rtc/rtc-stm32.c +++ b/drivers/rtc/rtc-stm32.c @@ -519,11 +519,7 @@ static int stm32_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) /* Write to Alarm register */ writel_relaxed(alrmar, rtc->base + regs->alrmar); - if (alrm->enabled) - stm32_rtc_alarm_irq_enable(dev, 1); - else - stm32_rtc_alarm_irq_enable(dev, 0); - + stm32_rtc_alarm_irq_enable(dev, alrm->enabled); end: stm32_rtc_wpr_lock(rtc); diff --git a/drivers/rtc/rtc-sun6i.c b/drivers/rtc/rtc-sun6i.c index 8128ec200ba2..c0e75c373605 100644 --- a/drivers/rtc/rtc-sun6i.c +++ b/drivers/rtc/rtc-sun6i.c @@ -672,6 +672,7 @@ static const struct of_device_id sun6i_rtc_dt_ids[] = { { .compatible = "allwinner,sun6i-a31-rtc" }, { .compatible = "allwinner,sun8i-a23-rtc" }, { .compatible = "allwinner,sun8i-h3-rtc" }, + { .compatible = "allwinner,sun8i-r40-rtc" }, { .compatible = "allwinner,sun8i-v3-rtc" }, { .compatible = "allwinner,sun50i-h5-rtc" }, { /* sentinel */ }, diff --git a/drivers/rtc/rtc-tegra.c b/drivers/rtc/rtc-tegra.c index f0ce76865434..8fa1b3febf69 100644 --- a/drivers/rtc/rtc-tegra.c +++ b/drivers/rtc/rtc-tegra.c @@ -2,7 +2,7 @@ /* * An RTC driver for the NVIDIA Tegra 200 series internal RTC. * - * Copyright (c) 2010, NVIDIA Corporation. + * Copyright (c) 2010-2019, NVIDIA Corporation. */ #include <linux/clk.h> @@ -18,10 +18,10 @@ #include <linux/rtc.h> #include <linux/slab.h> -/* set to 1 = busy every eight 32kHz clocks during copy of sec+msec to AHB */ +/* Set to 1 = busy every eight 32 kHz clocks during copy of sec+msec to AHB. */ #define TEGRA_RTC_REG_BUSY 0x004 #define TEGRA_RTC_REG_SECONDS 0x008 -/* when msec is read, the seconds are buffered into shadow seconds. */ +/* When msec is read, the seconds are buffered into shadow seconds. */ #define TEGRA_RTC_REG_SHADOW_SECONDS 0x00c #define TEGRA_RTC_REG_MILLI_SECONDS 0x010 #define TEGRA_RTC_REG_SECONDS_ALARM0 0x014 @@ -46,44 +46,48 @@ #define TEGRA_RTC_INTR_STATUS_SEC_ALARM0 (1<<0) struct tegra_rtc_info { - struct platform_device *pdev; - struct rtc_device *rtc_dev; - void __iomem *rtc_base; /* NULL if not initialized. */ - struct clk *clk; - int tegra_rtc_irq; /* alarm and periodic irq */ - spinlock_t tegra_rtc_lock; + struct platform_device *pdev; + struct rtc_device *rtc; + void __iomem *base; /* NULL if not initialized */ + struct clk *clk; + int irq; /* alarm and periodic IRQ */ + spinlock_t lock; }; -/* RTC hardware is busy when it is updating its values over AHB once - * every eight 32kHz clocks (~250uS). - * outside of these updates the CPU is free to write. - * CPU is always free to read. +/* + * RTC hardware is busy when it is updating its values over AHB once every + * eight 32 kHz clocks (~250 us). Outside of these updates the CPU is free to + * write. CPU is always free to read. */ static inline u32 tegra_rtc_check_busy(struct tegra_rtc_info *info) { - return readl(info->rtc_base + TEGRA_RTC_REG_BUSY) & 1; + return readl(info->base + TEGRA_RTC_REG_BUSY) & 1; } -/* Wait for hardware to be ready for writing. - * This function tries to maximize the amount of time before the next update. - * It does this by waiting for the RTC to become busy with its periodic update, - * then returning once the RTC first becomes not busy. +/* + * Wait for hardware to be ready for writing. This function tries to maximize + * the amount of time before the next update. It does this by waiting for the + * RTC to become busy with its periodic update, then returning once the RTC + * first becomes not busy. + * * This periodic update (where the seconds and milliseconds are copied to the - * AHB side) occurs every eight 32kHz clocks (~250uS). - * The behavior of this function allows us to make some assumptions without - * introducing a race, because 250uS is plenty of time to read/write a value. + * AHB side) occurs every eight 32 kHz clocks (~250 us). The behavior of this + * function allows us to make some assumptions without introducing a race, + * because 250 us is plenty of time to read/write a value. */ static int tegra_rtc_wait_while_busy(struct device *dev) { struct tegra_rtc_info *info = dev_get_drvdata(dev); + int retries = 500; /* ~490 us is the worst case, ~250 us is best */ - int retries = 500; /* ~490 us is the worst case, ~250 us is best. */ - - /* first wait for the RTC to become busy. this is when it - * posts its updated seconds+msec registers to AHB side. */ + /* + * First wait for the RTC to become busy. This is when it posts its + * updated seconds+msec registers to AHB side. + */ while (tegra_rtc_check_busy(info)) { if (!retries--) goto retry_failed; + udelay(1); } @@ -91,28 +95,30 @@ static int tegra_rtc_wait_while_busy(struct device *dev) return 0; retry_failed: - dev_err(dev, "write failed:retry count exceeded.\n"); + dev_err(dev, "write failed: retry count exceeded\n"); return -ETIMEDOUT; } static int tegra_rtc_read_time(struct device *dev, struct rtc_time *tm) { struct tegra_rtc_info *info = dev_get_drvdata(dev); - unsigned long sec, msec; - unsigned long sl_irq_flags; + unsigned long flags; + u32 sec, msec; - /* RTC hardware copies seconds to shadow seconds when a read - * of milliseconds occurs. use a lock to keep other threads out. */ - spin_lock_irqsave(&info->tegra_rtc_lock, sl_irq_flags); + /* + * RTC hardware copies seconds to shadow seconds when a read of + * milliseconds occurs. use a lock to keep other threads out. + */ + spin_lock_irqsave(&info->lock, flags); - msec = readl(info->rtc_base + TEGRA_RTC_REG_MILLI_SECONDS); - sec = readl(info->rtc_base + TEGRA_RTC_REG_SHADOW_SECONDS); + msec = readl(info->base + TEGRA_RTC_REG_MILLI_SECONDS); + sec = readl(info->base + TEGRA_RTC_REG_SHADOW_SECONDS); - spin_unlock_irqrestore(&info->tegra_rtc_lock, sl_irq_flags); + spin_unlock_irqrestore(&info->lock, flags); rtc_time64_to_tm(sec, tm); - dev_vdbg(dev, "time read as %lu. %ptR\n", sec, tm); + dev_vdbg(dev, "time read as %u, %ptR\n", sec, tm); return 0; } @@ -120,21 +126,21 @@ static int tegra_rtc_read_time(struct device *dev, struct rtc_time *tm) static int tegra_rtc_set_time(struct device *dev, struct rtc_time *tm) { struct tegra_rtc_info *info = dev_get_drvdata(dev); - unsigned long sec; + u32 sec; int ret; - /* convert tm to seconds. */ + /* convert tm to seconds */ sec = rtc_tm_to_time64(tm); - dev_vdbg(dev, "time set to %lu. %ptR\n", sec, tm); + dev_vdbg(dev, "time set to %u, %ptR\n", sec, tm); - /* seconds only written if wait succeeded. */ + /* seconds only written if wait succeeded */ ret = tegra_rtc_wait_while_busy(dev); if (!ret) - writel(sec, info->rtc_base + TEGRA_RTC_REG_SECONDS); + writel(sec, info->base + TEGRA_RTC_REG_SECONDS); dev_vdbg(dev, "time read back as %d\n", - readl(info->rtc_base + TEGRA_RTC_REG_SECONDS)); + readl(info->base + TEGRA_RTC_REG_SECONDS)); return ret; } @@ -142,22 +148,21 @@ static int tegra_rtc_set_time(struct device *dev, struct rtc_time *tm) static int tegra_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm) { struct tegra_rtc_info *info = dev_get_drvdata(dev); - unsigned long sec; - unsigned tmp; + u32 sec, value; - sec = readl(info->rtc_base + TEGRA_RTC_REG_SECONDS_ALARM0); + sec = readl(info->base + TEGRA_RTC_REG_SECONDS_ALARM0); if (sec == 0) { - /* alarm is disabled. */ + /* alarm is disabled */ alarm->enabled = 0; } else { - /* alarm is enabled. */ + /* alarm is enabled */ alarm->enabled = 1; rtc_time64_to_tm(sec, &alarm->time); } - tmp = readl(info->rtc_base + TEGRA_RTC_REG_INTR_STATUS); - alarm->pending = (tmp & TEGRA_RTC_INTR_STATUS_SEC_ALARM0) != 0; + value = readl(info->base + TEGRA_RTC_REG_INTR_STATUS); + alarm->pending = (value & TEGRA_RTC_INTR_STATUS_SEC_ALARM0) != 0; return 0; } @@ -165,22 +170,22 @@ static int tegra_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm) static int tegra_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled) { struct tegra_rtc_info *info = dev_get_drvdata(dev); - unsigned status; - unsigned long sl_irq_flags; + unsigned long flags; + u32 status; tegra_rtc_wait_while_busy(dev); - spin_lock_irqsave(&info->tegra_rtc_lock, sl_irq_flags); + spin_lock_irqsave(&info->lock, flags); - /* read the original value, and OR in the flag. */ - status = readl(info->rtc_base + TEGRA_RTC_REG_INTR_MASK); + /* read the original value, and OR in the flag */ + status = readl(info->base + TEGRA_RTC_REG_INTR_MASK); if (enabled) status |= TEGRA_RTC_INTR_MASK_SEC_ALARM0; /* set it */ else status &= ~TEGRA_RTC_INTR_MASK_SEC_ALARM0; /* clear it */ - writel(status, info->rtc_base + TEGRA_RTC_REG_INTR_MASK); + writel(status, info->base + TEGRA_RTC_REG_INTR_MASK); - spin_unlock_irqrestore(&info->tegra_rtc_lock, sl_irq_flags); + spin_unlock_irqrestore(&info->lock, flags); return 0; } @@ -188,7 +193,7 @@ static int tegra_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled) static int tegra_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) { struct tegra_rtc_info *info = dev_get_drvdata(dev); - unsigned long sec; + u32 sec; if (alarm->enabled) sec = rtc_tm_to_time64(&alarm->time); @@ -196,16 +201,16 @@ static int tegra_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) sec = 0; tegra_rtc_wait_while_busy(dev); - writel(sec, info->rtc_base + TEGRA_RTC_REG_SECONDS_ALARM0); + writel(sec, info->base + TEGRA_RTC_REG_SECONDS_ALARM0); dev_vdbg(dev, "alarm read back as %d\n", - readl(info->rtc_base + TEGRA_RTC_REG_SECONDS_ALARM0)); + readl(info->base + TEGRA_RTC_REG_SECONDS_ALARM0)); /* if successfully written and alarm is enabled ... */ if (sec) { tegra_rtc_alarm_irq_enable(dev, 1); - dev_vdbg(dev, "alarm set as %lu. %ptR\n", sec, &alarm->time); + dev_vdbg(dev, "alarm set as %u, %ptR\n", sec, &alarm->time); } else { - /* disable alarm if 0 or write error. */ + /* disable alarm if 0 or write error */ dev_vdbg(dev, "alarm disabled\n"); tegra_rtc_alarm_irq_enable(dev, 0); } @@ -227,39 +232,39 @@ static irqreturn_t tegra_rtc_irq_handler(int irq, void *data) { struct device *dev = data; struct tegra_rtc_info *info = dev_get_drvdata(dev); - unsigned long events = 0; - unsigned status; - unsigned long sl_irq_flags; + unsigned long events = 0, flags; + u32 status; - status = readl(info->rtc_base + TEGRA_RTC_REG_INTR_STATUS); + status = readl(info->base + TEGRA_RTC_REG_INTR_STATUS); if (status) { - /* clear the interrupt masks and status on any irq. */ + /* clear the interrupt masks and status on any IRQ */ tegra_rtc_wait_while_busy(dev); - spin_lock_irqsave(&info->tegra_rtc_lock, sl_irq_flags); - writel(0, info->rtc_base + TEGRA_RTC_REG_INTR_MASK); - writel(status, info->rtc_base + TEGRA_RTC_REG_INTR_STATUS); - spin_unlock_irqrestore(&info->tegra_rtc_lock, sl_irq_flags); + + spin_lock_irqsave(&info->lock, flags); + writel(0, info->base + TEGRA_RTC_REG_INTR_MASK); + writel(status, info->base + TEGRA_RTC_REG_INTR_STATUS); + spin_unlock_irqrestore(&info->lock, flags); } - /* check if Alarm */ - if ((status & TEGRA_RTC_INTR_STATUS_SEC_ALARM0)) + /* check if alarm */ + if (status & TEGRA_RTC_INTR_STATUS_SEC_ALARM0) events |= RTC_IRQF | RTC_AF; - /* check if Periodic */ - if ((status & TEGRA_RTC_INTR_STATUS_SEC_CDN_ALARM)) + /* check if periodic */ + if (status & TEGRA_RTC_INTR_STATUS_SEC_CDN_ALARM) events |= RTC_IRQF | RTC_PF; - rtc_update_irq(info->rtc_dev, 1, events); + rtc_update_irq(info->rtc, 1, events); return IRQ_HANDLED; } static const struct rtc_class_ops tegra_rtc_ops = { - .read_time = tegra_rtc_read_time, - .set_time = tegra_rtc_set_time, - .read_alarm = tegra_rtc_read_alarm, - .set_alarm = tegra_rtc_set_alarm, - .proc = tegra_rtc_proc, + .read_time = tegra_rtc_read_time, + .set_time = tegra_rtc_set_time, + .read_alarm = tegra_rtc_read_alarm, + .set_alarm = tegra_rtc_set_alarm, + .proc = tegra_rtc_proc, .alarm_irq_enable = tegra_rtc_alarm_irq_enable, }; @@ -269,21 +274,20 @@ static const struct of_device_id tegra_rtc_dt_match[] = { }; MODULE_DEVICE_TABLE(of, tegra_rtc_dt_match); -static int __init tegra_rtc_probe(struct platform_device *pdev) +static int tegra_rtc_probe(struct platform_device *pdev) { struct tegra_rtc_info *info; struct resource *res; int ret; - info = devm_kzalloc(&pdev->dev, sizeof(struct tegra_rtc_info), - GFP_KERNEL); + info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - info->rtc_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(info->rtc_base)) - return PTR_ERR(info->rtc_base); + info->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(info->base)) + return PTR_ERR(info->base); ret = platform_get_irq(pdev, 0); if (ret <= 0) { @@ -291,14 +295,14 @@ static int __init tegra_rtc_probe(struct platform_device *pdev) return ret; } - info->tegra_rtc_irq = ret; + info->irq = ret; - info->rtc_dev = devm_rtc_allocate_device(&pdev->dev); - if (IS_ERR(info->rtc_dev)) - return PTR_ERR(info->rtc_dev); + info->rtc = devm_rtc_allocate_device(&pdev->dev); + if (IS_ERR(info->rtc)) + return PTR_ERR(info->rtc); - info->rtc_dev->ops = &tegra_rtc_ops; - info->rtc_dev->range_max = U32_MAX; + info->rtc->ops = &tegra_rtc_ops; + info->rtc->range_max = U32_MAX; info->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(info->clk)) @@ -308,33 +312,30 @@ static int __init tegra_rtc_probe(struct platform_device *pdev) if (ret < 0) return ret; - /* set context info. */ + /* set context info */ info->pdev = pdev; - spin_lock_init(&info->tegra_rtc_lock); + spin_lock_init(&info->lock); platform_set_drvdata(pdev, info); - /* clear out the hardware. */ - writel(0, info->rtc_base + TEGRA_RTC_REG_SECONDS_ALARM0); - writel(0xffffffff, info->rtc_base + TEGRA_RTC_REG_INTR_STATUS); - writel(0, info->rtc_base + TEGRA_RTC_REG_INTR_MASK); + /* clear out the hardware */ + writel(0, info->base + TEGRA_RTC_REG_SECONDS_ALARM0); + writel(0xffffffff, info->base + TEGRA_RTC_REG_INTR_STATUS); + writel(0, info->base + TEGRA_RTC_REG_INTR_MASK); device_init_wakeup(&pdev->dev, 1); - ret = devm_request_irq(&pdev->dev, info->tegra_rtc_irq, - tegra_rtc_irq_handler, IRQF_TRIGGER_HIGH, - dev_name(&pdev->dev), &pdev->dev); + ret = devm_request_irq(&pdev->dev, info->irq, tegra_rtc_irq_handler, + IRQF_TRIGGER_HIGH, dev_name(&pdev->dev), + &pdev->dev); if (ret) { - dev_err(&pdev->dev, - "Unable to request interrupt for device (err=%d).\n", - ret); + dev_err(&pdev->dev, "failed to request interrupt: %d\n", ret); goto disable_clk; } - ret = rtc_register_device(info->rtc_dev); + ret = rtc_register_device(info->rtc); if (ret) { - dev_err(&pdev->dev, "Unable to register device (err=%d).\n", - ret); + dev_err(&pdev->dev, "failed to register device: %d\n", ret); goto disable_clk; } @@ -363,20 +364,20 @@ static int tegra_rtc_suspend(struct device *dev) tegra_rtc_wait_while_busy(dev); - /* only use ALARM0 as a wake source. */ - writel(0xffffffff, info->rtc_base + TEGRA_RTC_REG_INTR_STATUS); + /* only use ALARM0 as a wake source */ + writel(0xffffffff, info->base + TEGRA_RTC_REG_INTR_STATUS); writel(TEGRA_RTC_INTR_STATUS_SEC_ALARM0, - info->rtc_base + TEGRA_RTC_REG_INTR_MASK); + info->base + TEGRA_RTC_REG_INTR_MASK); dev_vdbg(dev, "alarm sec = %d\n", - readl(info->rtc_base + TEGRA_RTC_REG_SECONDS_ALARM0)); + readl(info->base + TEGRA_RTC_REG_SECONDS_ALARM0)); - dev_vdbg(dev, "Suspend (device_may_wakeup=%d) irq:%d\n", - device_may_wakeup(dev), info->tegra_rtc_irq); + dev_vdbg(dev, "Suspend (device_may_wakeup=%d) IRQ:%d\n", + device_may_wakeup(dev), info->irq); - /* leave the alarms on as a wake source. */ + /* leave the alarms on as a wake source */ if (device_may_wakeup(dev)) - enable_irq_wake(info->tegra_rtc_irq); + enable_irq_wake(info->irq); return 0; } @@ -386,10 +387,11 @@ static int tegra_rtc_resume(struct device *dev) struct tegra_rtc_info *info = dev_get_drvdata(dev); dev_vdbg(dev, "Resume (device_may_wakeup=%d)\n", - device_may_wakeup(dev)); - /* alarms were left on as a wake source, turn them off. */ + device_may_wakeup(dev)); + + /* alarms were left on as a wake source, turn them off */ if (device_may_wakeup(dev)) - disable_irq_wake(info->tegra_rtc_irq); + disable_irq_wake(info->irq); return 0; } @@ -399,22 +401,21 @@ static SIMPLE_DEV_PM_OPS(tegra_rtc_pm_ops, tegra_rtc_suspend, tegra_rtc_resume); static void tegra_rtc_shutdown(struct platform_device *pdev) { - dev_vdbg(&pdev->dev, "disabling interrupts.\n"); + dev_vdbg(&pdev->dev, "disabling interrupts\n"); tegra_rtc_alarm_irq_enable(&pdev->dev, 0); } -MODULE_ALIAS("platform:tegra_rtc"); static struct platform_driver tegra_rtc_driver = { - .remove = tegra_rtc_remove, - .shutdown = tegra_rtc_shutdown, - .driver = { - .name = "tegra_rtc", + .probe = tegra_rtc_probe, + .remove = tegra_rtc_remove, + .shutdown = tegra_rtc_shutdown, + .driver = { + .name = "tegra_rtc", .of_match_table = tegra_rtc_dt_match, - .pm = &tegra_rtc_pm_ops, + .pm = &tegra_rtc_pm_ops, }, }; - -module_platform_driver_probe(tegra_rtc_driver, tegra_rtc_probe); +module_platform_driver(tegra_rtc_driver); MODULE_AUTHOR("Jon Mayo <jmayo@nvidia.com>"); MODULE_DESCRIPTION("driver for Tegra internal RTC"); diff --git a/drivers/rtc/rtc-test.c b/drivers/rtc/rtc-test.c index b298e9902f45..74b3a0603b73 100644 --- a/drivers/rtc/rtc-test.c +++ b/drivers/rtc/rtc-test.c @@ -133,6 +133,7 @@ static int test_probe(struct platform_device *plat_dev) break; default: rtd->rtc->ops = &test_rtc_ops; + device_init_wakeup(&plat_dev->dev, 1); } timer_setup(&rtd->alarm, test_rtc_alarm_handler, 0); diff --git a/drivers/rtc/rtc-tps65910.c b/drivers/rtc/rtc-tps65910.c index 8556d925e52a..7078f6da1cbc 100644 --- a/drivers/rtc/rtc-tps65910.c +++ b/drivers/rtc/rtc-tps65910.c @@ -143,7 +143,7 @@ static int tps65910_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm) struct tps65910 *tps = dev_get_drvdata(dev->parent); int ret; - ret = regmap_bulk_read(tps->regmap, TPS65910_SECONDS, alarm_data, + ret = regmap_bulk_read(tps->regmap, TPS65910_ALARM_SECONDS, alarm_data, NUM_TIME_REGS); if (ret < 0) { dev_err(dev, "rtc_read_alarm error %d\n", ret); diff --git a/drivers/rtc/rtc-wm831x.c b/drivers/rtc/rtc-wm831x.c index d2e8b21c90c4..ccef887d2690 100644 --- a/drivers/rtc/rtc-wm831x.c +++ b/drivers/rtc/rtc-wm831x.c @@ -435,7 +435,8 @@ static int wm831x_rtc_probe(struct platform_device *pdev) ret = devm_request_threaded_irq(&pdev->dev, alm_irq, NULL, wm831x_alm_irq, - IRQF_TRIGGER_RISING, "RTC alarm", + IRQF_TRIGGER_RISING | IRQF_ONESHOT, + "RTC alarm", wm831x_rtc); if (ret != 0) { dev_err(&pdev->dev, "Failed to request alarm IRQ %d: %d\n", diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index d04d4378ca50..63502ca537eb 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -679,7 +679,7 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char goto put_dev; dev_info->dax_dev = alloc_dax(dev_info, dev_info->gd->disk_name, - &dcssblk_dax_ops); + &dcssblk_dax_ops, DAXDEV_F_SYNC); if (!dev_info->dax_dev) { rc = -ENOMEM; goto put_dev; diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 1705398b026a..297e1076e571 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -792,7 +792,7 @@ static int virtscsi_probe(struct virtio_device *vdev) num_targets = virtscsi_config_get(vdev, max_target) + 1; shost = scsi_host_alloc(&virtscsi_host_template, - sizeof(*vscsi) + sizeof(vscsi->req_vqs[0]) * num_queues); + struct_size(vscsi, req_vqs, num_queues)); if (!shost) return -ENOMEM; diff --git a/drivers/soc/qcom/mdt_loader.c b/drivers/soc/qcom/mdt_loader.c index 9ca7d9484de0..24cd193dec55 100644 --- a/drivers/soc/qcom/mdt_loader.c +++ b/drivers/soc/qcom/mdt_loader.c @@ -66,6 +66,66 @@ ssize_t qcom_mdt_get_size(const struct firmware *fw) } EXPORT_SYMBOL_GPL(qcom_mdt_get_size); +/** + * qcom_mdt_read_metadata() - read header and metadata from mdt or mbn + * @fw: firmware of mdt header or mbn + * @data_len: length of the read metadata blob + * + * The mechanism that performs the authentication of the loading firmware + * expects an ELF header directly followed by the segment of hashes, with no + * padding inbetween. This function allocates a chunk of memory for this pair + * and copy the two pieces into the buffer. + * + * In the case of split firmware the hash is found directly following the ELF + * header, rather than at p_offset described by the second program header. + * + * The caller is responsible to free (kfree()) the returned pointer. + * + * Return: pointer to data, or ERR_PTR() + */ +void *qcom_mdt_read_metadata(const struct firmware *fw, size_t *data_len) +{ + const struct elf32_phdr *phdrs; + const struct elf32_hdr *ehdr; + size_t hash_offset; + size_t hash_size; + size_t ehdr_size; + void *data; + + ehdr = (struct elf32_hdr *)fw->data; + phdrs = (struct elf32_phdr *)(ehdr + 1); + + if (ehdr->e_phnum < 2) + return ERR_PTR(-EINVAL); + + if (phdrs[0].p_type == PT_LOAD || phdrs[1].p_type == PT_LOAD) + return ERR_PTR(-EINVAL); + + if ((phdrs[1].p_flags & QCOM_MDT_TYPE_MASK) != QCOM_MDT_TYPE_HASH) + return ERR_PTR(-EINVAL); + + ehdr_size = phdrs[0].p_filesz; + hash_size = phdrs[1].p_filesz; + + data = kmalloc(ehdr_size + hash_size, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + + /* Is the header and hash already packed */ + if (ehdr_size + hash_size == fw->size) + hash_offset = phdrs[0].p_filesz; + else + hash_offset = phdrs[1].p_offset; + + memcpy(data, fw->data, ehdr_size); + memcpy(data + ehdr_size, fw->data + hash_offset, hash_size); + + *data_len = ehdr_size + hash_size; + + return data; +} +EXPORT_SYMBOL_GPL(qcom_mdt_read_metadata); + static int __qcom_mdt_load(struct device *dev, const struct firmware *fw, const char *firmware, int pas_id, void *mem_region, phys_addr_t mem_phys, size_t mem_size, @@ -78,12 +138,14 @@ static int __qcom_mdt_load(struct device *dev, const struct firmware *fw, phys_addr_t mem_reloc; phys_addr_t min_addr = PHYS_ADDR_MAX; phys_addr_t max_addr = 0; + size_t metadata_len; size_t fw_name_len; ssize_t offset; + void *metadata; char *fw_name; bool relocate = false; void *ptr; - int ret; + int ret = 0; int i; if (!fw || !mem_region || !mem_phys || !mem_size) @@ -101,7 +163,15 @@ static int __qcom_mdt_load(struct device *dev, const struct firmware *fw, return -ENOMEM; if (pas_init) { - ret = qcom_scm_pas_init_image(pas_id, fw->data, fw->size); + metadata = qcom_mdt_read_metadata(fw, &metadata_len); + if (IS_ERR(metadata)) { + ret = PTR_ERR(metadata); + goto out; + } + + ret = qcom_scm_pas_init_image(pas_id, metadata, metadata_len); + + kfree(metadata); if (ret) { dev_err(dev, "invalid firmware metadata\n"); goto out; @@ -162,7 +232,19 @@ static int __qcom_mdt_load(struct device *dev, const struct firmware *fw, ptr = mem_region + offset; - if (phdr->p_filesz) { + if (phdr->p_filesz && phdr->p_offset < fw->size) { + /* Firmware is large enough to be non-split */ + if (phdr->p_offset + phdr->p_filesz > fw->size) { + dev_err(dev, + "failed to load segment %d from truncated file %s\n", + i, firmware); + ret = -EINVAL; + break; + } + + memcpy(ptr, fw->data + phdr->p_offset, phdr->p_filesz); + } else if (phdr->p_filesz) { + /* Firmware not large enough, load split-out segments */ sprintf(fw_name + fw_name_len - 3, "b%02d", i); ret = request_firmware_into_buf(&seg_fw, fw_name, dev, ptr, phdr->p_filesz); diff --git a/drivers/soc/tegra/fuse/fuse-tegra20.c b/drivers/soc/tegra/fuse/fuse-tegra20.c index 4bb16e9bc297..d4aef9c4a94c 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra20.c +++ b/drivers/soc/tegra/fuse/fuse-tegra20.c @@ -99,7 +99,7 @@ static int tegra20_fuse_probe(struct tegra_fuse *fuse) dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); - fuse->apbdma.chan = __dma_request_channel(&mask, dma_filter, NULL); + fuse->apbdma.chan = dma_request_channel(mask, dma_filter, NULL); if (!fuse->apbdma.chan) return -EPROBE_DEFER; diff --git a/drivers/thermal/fair_share.c b/drivers/thermal/fair_share.c index 8ff109fb77e1..afd99f668c65 100644 --- a/drivers/thermal/fair_share.c +++ b/drivers/thermal/fair_share.c @@ -117,14 +117,4 @@ static struct thermal_governor thermal_gov_fair_share = { .name = "fair_share", .throttle = fair_share_throttle, }; - -int thermal_gov_fair_share_register(void) -{ - return thermal_register_governor(&thermal_gov_fair_share); -} - -void thermal_gov_fair_share_unregister(void) -{ - thermal_unregister_governor(&thermal_gov_fair_share); -} - +THERMAL_GOVERNOR_DECLARE(thermal_gov_fair_share); diff --git a/drivers/thermal/gov_bang_bang.c b/drivers/thermal/gov_bang_bang.c index 632fb925e975..e0575d29023a 100644 --- a/drivers/thermal/gov_bang_bang.c +++ b/drivers/thermal/gov_bang_bang.c @@ -116,13 +116,4 @@ static struct thermal_governor thermal_gov_bang_bang = { .name = "bang_bang", .throttle = bang_bang_control, }; - -int thermal_gov_bang_bang_register(void) -{ - return thermal_register_governor(&thermal_gov_bang_bang); -} - -void thermal_gov_bang_bang_unregister(void) -{ - thermal_unregister_governor(&thermal_gov_bang_bang); -} +THERMAL_GOVERNOR_DECLARE(thermal_gov_bang_bang); diff --git a/drivers/thermal/intel/int340x_thermal/Kconfig b/drivers/thermal/intel/int340x_thermal/Kconfig index 5333e018c88c..797907542e43 100644 --- a/drivers/thermal/intel/int340x_thermal/Kconfig +++ b/drivers/thermal/intel/int340x_thermal/Kconfig @@ -40,4 +40,10 @@ config INT3406_THERMAL brightness in order to address a thermal condition or to reduce power consumed by display device. +config PROC_THERMAL_MMIO_RAPL + bool + depends on 64BIT + depends on POWERCAP + select INTEL_RAPL_CORE + default y endif diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c index 53c84fa498ce..213ab3cc6b80 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c @@ -11,6 +11,8 @@ #include <linux/platform_device.h> #include <linux/acpi.h> #include <linux/thermal.h> +#include <linux/cpuhotplug.h> +#include <linux/intel_rapl.h> #include "int340x_thermal_zone.h" #include "../intel_soc_dts_iosf.h" @@ -37,6 +39,8 @@ /* GeminiLake thermal reporting device */ #define PCI_DEVICE_ID_PROC_GLK_THERMAL 0x318C +#define DRV_NAME "proc_thermal" + struct power_config { u32 index; u32 min_uw; @@ -52,6 +56,7 @@ struct proc_thermal_device { struct power_config power_limits[2]; struct int34x_thermal_zone *int340x_zone; struct intel_soc_dts_sensors *soc_dts; + void __iomem *mmio_base; }; enum proc_thermal_emum_mode_type { @@ -60,6 +65,12 @@ enum proc_thermal_emum_mode_type { PROC_THERMAL_PLATFORM_DEV }; +struct rapl_mmio_regs { + u64 reg_unit; + u64 regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; + int limits[RAPL_DOMAIN_MAX]; +}; + /* * We can have only one type of enumeration, PCI or Platform, * not both. So we don't need instance specific data. @@ -367,8 +378,151 @@ static irqreturn_t proc_thermal_pci_msi_irq(int irq, void *devid) return IRQ_HANDLED; } +#ifdef CONFIG_PROC_THERMAL_MMIO_RAPL + +#define MCHBAR 0 + +/* RAPL Support via MMIO interface */ +static struct rapl_if_priv rapl_mmio_priv; + +static int rapl_mmio_cpu_online(unsigned int cpu) +{ + struct rapl_package *rp; + + /* mmio rapl supports package 0 only for now */ + if (topology_physical_package_id(cpu)) + return 0; + + rp = rapl_find_package_domain(cpu, &rapl_mmio_priv); + if (!rp) { + rp = rapl_add_package(cpu, &rapl_mmio_priv); + if (IS_ERR(rp)) + return PTR_ERR(rp); + } + cpumask_set_cpu(cpu, &rp->cpumask); + return 0; +} + +static int rapl_mmio_cpu_down_prep(unsigned int cpu) +{ + struct rapl_package *rp; + int lead_cpu; + + rp = rapl_find_package_domain(cpu, &rapl_mmio_priv); + if (!rp) + return 0; + + cpumask_clear_cpu(cpu, &rp->cpumask); + lead_cpu = cpumask_first(&rp->cpumask); + if (lead_cpu >= nr_cpu_ids) + rapl_remove_package(rp); + else if (rp->lead_cpu == cpu) + rp->lead_cpu = lead_cpu; + return 0; +} + +static int rapl_mmio_read_raw(int cpu, struct reg_action *ra) +{ + if (!ra->reg) + return -EINVAL; + + ra->value = readq((void __iomem *)ra->reg); + ra->value &= ra->mask; + return 0; +} + +static int rapl_mmio_write_raw(int cpu, struct reg_action *ra) +{ + u64 val; + + if (!ra->reg) + return -EINVAL; + + val = readq((void __iomem *)ra->reg); + val &= ~ra->mask; + val |= ra->value; + writeq(val, (void __iomem *)ra->reg); + return 0; +} + +static int proc_thermal_rapl_add(struct pci_dev *pdev, + struct proc_thermal_device *proc_priv, + struct rapl_mmio_regs *rapl_regs) +{ + enum rapl_domain_reg_id reg; + enum rapl_domain_type domain; + int ret; + + if (!rapl_regs) + return 0; + + ret = pcim_iomap_regions(pdev, 1 << MCHBAR, DRV_NAME); + if (ret) { + dev_err(&pdev->dev, "cannot reserve PCI memory region\n"); + return -ENOMEM; + } + + proc_priv->mmio_base = pcim_iomap_table(pdev)[MCHBAR]; + + for (domain = RAPL_DOMAIN_PACKAGE; domain < RAPL_DOMAIN_MAX; domain++) { + for (reg = RAPL_DOMAIN_REG_LIMIT; reg < RAPL_DOMAIN_REG_MAX; reg++) + if (rapl_regs->regs[domain][reg]) + rapl_mmio_priv.regs[domain][reg] = + (u64)proc_priv->mmio_base + + rapl_regs->regs[domain][reg]; + rapl_mmio_priv.limits[domain] = rapl_regs->limits[domain]; + } + rapl_mmio_priv.reg_unit = (u64)proc_priv->mmio_base + rapl_regs->reg_unit; + + rapl_mmio_priv.read_raw = rapl_mmio_read_raw; + rapl_mmio_priv.write_raw = rapl_mmio_write_raw; + + rapl_mmio_priv.control_type = powercap_register_control_type(NULL, "intel-rapl-mmio", NULL); + if (IS_ERR(rapl_mmio_priv.control_type)) { + pr_debug("failed to register powercap control_type.\n"); + return PTR_ERR(rapl_mmio_priv.control_type); + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online", + rapl_mmio_cpu_online, rapl_mmio_cpu_down_prep); + if (ret < 0) { + powercap_unregister_control_type(rapl_mmio_priv.control_type); + return ret; + } + rapl_mmio_priv.pcap_rapl_online = ret; + + return 0; +} + +static void proc_thermal_rapl_remove(void) +{ + cpuhp_remove_state(rapl_mmio_priv.pcap_rapl_online); + powercap_unregister_control_type(rapl_mmio_priv.control_type); +} + +static const struct rapl_mmio_regs rapl_mmio_hsw = { + .reg_unit = 0x5938, + .regs[RAPL_DOMAIN_PACKAGE] = { 0x59a0, 0x593c, 0x58f0, 0, 0x5930}, + .regs[RAPL_DOMAIN_DRAM] = { 0x58e0, 0x58e8, 0x58ec, 0, 0}, + .limits[RAPL_DOMAIN_PACKAGE] = 2, + .limits[RAPL_DOMAIN_DRAM] = 2, +}; + +#else + +static int proc_thermal_rapl_add(struct pci_dev *pdev, + struct proc_thermal_device *proc_priv, + struct rapl_mmio_regs *rapl_regs) +{ + return 0; +} +static void proc_thermal_rapl_remove(void) {} +static const struct rapl_mmio_regs rapl_mmio_hsw; + +#endif /* CONFIG_MMIO_RAPL */ + static int proc_thermal_pci_probe(struct pci_dev *pdev, - const struct pci_device_id *unused) + const struct pci_device_id *id) { struct proc_thermal_device *proc_priv; int ret; @@ -378,15 +532,21 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, return -ENODEV; } - ret = pci_enable_device(pdev); + ret = pcim_enable_device(pdev); if (ret < 0) { dev_err(&pdev->dev, "error: could not enable device\n"); return ret; } ret = proc_thermal_add(&pdev->dev, &proc_priv); + if (ret) + return ret; + + ret = proc_thermal_rapl_add(pdev, proc_priv, + (struct rapl_mmio_regs *)id->driver_data); if (ret) { - pci_disable_device(pdev); + dev_err(&pdev->dev, "failed to add RAPL MMIO interface\n"); + proc_thermal_remove(proc_priv); return ret; } @@ -439,14 +599,31 @@ static void proc_thermal_pci_remove(struct pci_dev *pdev) pci_disable_msi(pdev); } } + proc_thermal_rapl_remove(); proc_thermal_remove(proc_priv); - pci_disable_device(pdev); } +#ifdef CONFIG_PM_SLEEP +static int proc_thermal_resume(struct device *dev) +{ + struct proc_thermal_device *proc_dev; + + proc_dev = dev_get_drvdata(dev); + proc_thermal_read_ppcc(proc_dev); + + return 0; +} +#else +#define proc_thermal_resume NULL +#endif + +static SIMPLE_DEV_PM_OPS(proc_thermal_pm, NULL, proc_thermal_resume); + static const struct pci_device_id proc_thermal_pci_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BDW_THERMAL)}, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_HSB_THERMAL)}, - { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_SKL_THERMAL)}, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_SKL_THERMAL), + .driver_data = (kernel_ulong_t)&rapl_mmio_hsw, }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BSW_THERMAL)}, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BXT0_THERMAL)}, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PROC_BXT1_THERMAL)}, @@ -461,10 +638,11 @@ static const struct pci_device_id proc_thermal_pci_ids[] = { MODULE_DEVICE_TABLE(pci, proc_thermal_pci_ids); static struct pci_driver proc_thermal_pci_driver = { - .name = "proc_thermal", + .name = DRV_NAME, .probe = proc_thermal_pci_probe, .remove = proc_thermal_pci_remove, .id_table = proc_thermal_pci_ids, + .driver.pm = &proc_thermal_pm, }; static const struct acpi_device_id int3401_device_ids[] = { @@ -479,6 +657,7 @@ static struct platform_driver int3401_driver = { .driver = { .name = "int3401 thermal", .acpi_match_table = int3401_device_ids, + .pm = &proc_thermal_pm, }, }; diff --git a/drivers/thermal/power_allocator.c b/drivers/thermal/power_allocator.c index 3055f9a12a17..44636475b2a3 100644 --- a/drivers/thermal/power_allocator.c +++ b/drivers/thermal/power_allocator.c @@ -651,13 +651,4 @@ static struct thermal_governor thermal_gov_power_allocator = { .unbind_from_tz = power_allocator_unbind, .throttle = power_allocator_throttle, }; - -int thermal_gov_power_allocator_register(void) -{ - return thermal_register_governor(&thermal_gov_power_allocator); -} - -void thermal_gov_power_allocator_unregister(void) -{ - thermal_unregister_governor(&thermal_gov_power_allocator); -} +THERMAL_GOVERNOR_DECLARE(thermal_gov_power_allocator); diff --git a/drivers/thermal/step_wise.c b/drivers/thermal/step_wise.c index 81a183befd48..6e051cbd824f 100644 --- a/drivers/thermal/step_wise.c +++ b/drivers/thermal/step_wise.c @@ -206,13 +206,4 @@ static struct thermal_governor thermal_gov_step_wise = { .name = "step_wise", .throttle = step_wise_throttle, }; - -int thermal_gov_step_wise_register(void) -{ - return thermal_register_governor(&thermal_gov_step_wise); -} - -void thermal_gov_step_wise_unregister(void) -{ - thermal_unregister_governor(&thermal_gov_step_wise); -} +THERMAL_GOVERNOR_DECLARE(thermal_gov_step_wise); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 46cfb7de4eb2..6bab66e84eb5 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -243,36 +243,42 @@ int thermal_build_list_of_policies(char *buf) return count; } -static int __init thermal_register_governors(void) +static void __init thermal_unregister_governors(void) { - int result; + struct thermal_governor **governor; - result = thermal_gov_step_wise_register(); - if (result) - return result; + for_each_governor_table(governor) + thermal_unregister_governor(*governor); +} - result = thermal_gov_fair_share_register(); - if (result) - return result; +static int __init thermal_register_governors(void) +{ + int ret = 0; + struct thermal_governor **governor; - result = thermal_gov_bang_bang_register(); - if (result) - return result; + for_each_governor_table(governor) { + ret = thermal_register_governor(*governor); + if (ret) { + pr_err("Failed to register governor: '%s'", + (*governor)->name); + break; + } - result = thermal_gov_user_space_register(); - if (result) - return result; + pr_info("Registered thermal governor '%s'", + (*governor)->name); + } - return thermal_gov_power_allocator_register(); -} + if (ret) { + struct thermal_governor **gov; -static void __init thermal_unregister_governors(void) -{ - thermal_gov_step_wise_unregister(); - thermal_gov_fair_share_unregister(); - thermal_gov_bang_bang_unregister(); - thermal_gov_user_space_unregister(); - thermal_gov_power_allocator_unregister(); + for_each_governor_table(gov) { + if (gov == governor) + break; + thermal_unregister_governor(*gov); + } + } + + return ret; } /* diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index 0df190ed82a7..207b0cda70da 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -15,6 +15,21 @@ /* Initial state of a cooling device during binding */ #define THERMAL_NO_TARGET -1UL +/* Init section thermal table */ +extern struct thermal_governor *__governor_thermal_table[]; +extern struct thermal_governor *__governor_thermal_table_end[]; + +#define THERMAL_TABLE_ENTRY(table, name) \ + static typeof(name) *__thermal_table_entry_##name \ + __used __section(__##table##_thermal_table) = &name + +#define THERMAL_GOVERNOR_DECLARE(name) THERMAL_TABLE_ENTRY(governor, name) + +#define for_each_governor_table(__governor) \ + for (__governor = __governor_thermal_table; \ + __governor < __governor_thermal_table_end; \ + __governor++) + /* * This structure is used to describe the behavior of * a certain cooling device on a certain trip point @@ -74,46 +89,6 @@ thermal_cooling_device_stats_update(struct thermal_cooling_device *cdev, unsigned long new_state) {} #endif /* CONFIG_THERMAL_STATISTICS */ -#ifdef CONFIG_THERMAL_GOV_STEP_WISE -int thermal_gov_step_wise_register(void); -void thermal_gov_step_wise_unregister(void); -#else -static inline int thermal_gov_step_wise_register(void) { return 0; } -static inline void thermal_gov_step_wise_unregister(void) {} -#endif /* CONFIG_THERMAL_GOV_STEP_WISE */ - -#ifdef CONFIG_THERMAL_GOV_FAIR_SHARE -int thermal_gov_fair_share_register(void); -void thermal_gov_fair_share_unregister(void); -#else -static inline int thermal_gov_fair_share_register(void) { return 0; } -static inline void thermal_gov_fair_share_unregister(void) {} -#endif /* CONFIG_THERMAL_GOV_FAIR_SHARE */ - -#ifdef CONFIG_THERMAL_GOV_BANG_BANG -int thermal_gov_bang_bang_register(void); -void thermal_gov_bang_bang_unregister(void); -#else -static inline int thermal_gov_bang_bang_register(void) { return 0; } -static inline void thermal_gov_bang_bang_unregister(void) {} -#endif /* CONFIG_THERMAL_GOV_BANG_BANG */ - -#ifdef CONFIG_THERMAL_GOV_USER_SPACE -int thermal_gov_user_space_register(void); -void thermal_gov_user_space_unregister(void); -#else -static inline int thermal_gov_user_space_register(void) { return 0; } -static inline void thermal_gov_user_space_unregister(void) {} -#endif /* CONFIG_THERMAL_GOV_USER_SPACE */ - -#ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR -int thermal_gov_power_allocator_register(void); -void thermal_gov_power_allocator_unregister(void); -#else -static inline int thermal_gov_power_allocator_register(void) { return 0; } -static inline void thermal_gov_power_allocator_unregister(void) {} -#endif /* CONFIG_THERMAL_GOV_POWER_ALLOCATOR */ - /* device tree support */ #ifdef CONFIG_THERMAL_OF int of_parse_thermal_zones(void); diff --git a/drivers/thermal/user_space.c b/drivers/thermal/user_space.c index d62a99fc60a6..962873fd9242 100644 --- a/drivers/thermal/user_space.c +++ b/drivers/thermal/user_space.c @@ -44,14 +44,4 @@ static struct thermal_governor thermal_gov_user_space = { .name = "user_space", .throttle = notify_user_space, }; - -int thermal_gov_user_space_register(void) -{ - return thermal_register_governor(&thermal_gov_user_space); -} - -void thermal_gov_user_space_unregister(void) -{ - thermal_unregister_governor(&thermal_gov_user_space); -} - +THERMAL_GOVERNOR_DECLARE(thermal_gov_user_space); diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c index ed8608763134..b558d4cfd082 100644 --- a/drivers/vfio/mdev/mdev_core.c +++ b/drivers/vfio/mdev/mdev_core.c @@ -143,6 +143,8 @@ int mdev_register_device(struct device *dev, const struct mdev_parent_ops *ops) { int ret; struct mdev_parent *parent; + char *env_string = "MDEV_STATE=registered"; + char *envp[] = { env_string, NULL }; /* check for mandatory ops */ if (!ops || !ops->create || !ops->remove || !ops->supported_type_groups) @@ -194,6 +196,8 @@ int mdev_register_device(struct device *dev, const struct mdev_parent_ops *ops) mutex_unlock(&parent_list_lock); dev_info(dev, "MDEV: Registered\n"); + kobject_uevent_env(&dev->kobj, KOBJ_CHANGE, envp); + return 0; add_dev_err: @@ -217,6 +221,8 @@ EXPORT_SYMBOL(mdev_register_device); void mdev_unregister_device(struct device *dev) { struct mdev_parent *parent; + char *env_string = "MDEV_STATE=unregistered"; + char *envp[] = { env_string, NULL }; mutex_lock(&parent_list_lock); parent = __find_parent_device(dev); @@ -240,6 +246,9 @@ void mdev_unregister_device(struct device *dev) up_write(&parent->unreg_sem); mdev_put_parent(parent); + + /* We still have the caller's reference to use for the uevent */ + kobject_uevent_env(&dev->kobj, KOBJ_CHANGE, envp); } EXPORT_SYMBOL(mdev_unregister_device); diff --git a/drivers/vfio/pci/vfio_pci_nvlink2.c b/drivers/vfio/pci/vfio_pci_nvlink2.c index 50fe3c4f7feb..f2983f0f84be 100644 --- a/drivers/vfio/pci/vfio_pci_nvlink2.c +++ b/drivers/vfio/pci/vfio_pci_nvlink2.c @@ -161,8 +161,7 @@ static int vfio_pci_nvgpu_mmap(struct vfio_pci_device *vdev, atomic_inc(&data->mm->mm_count); ret = (int) mm_iommu_newdev(data->mm, data->useraddr, - (vma->vm_end - vma->vm_start) >> PAGE_SHIFT, - data->gpu_hpa, &data->mem); + vma_pages(vma), data->gpu_hpa, &data->mem); trace_vfio_pci_nvgpu_mmap(vdev->pdev, data->gpu_hpa, data->useraddr, vma->vm_end - vma->vm_start, ret); diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c index 7048c9198c21..8ce9ad21129f 100644 --- a/drivers/vfio/vfio_iommu_spapr_tce.c +++ b/drivers/vfio/vfio_iommu_spapr_tce.c @@ -19,6 +19,7 @@ #include <linux/vmalloc.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> +#include <linux/mm.h> #include <asm/iommu.h> #include <asm/tce.h> @@ -31,51 +32,6 @@ static void tce_iommu_detach_group(void *iommu_data, struct iommu_group *iommu_group); -static long try_increment_locked_vm(struct mm_struct *mm, long npages) -{ - long ret = 0, locked, lock_limit; - - if (WARN_ON_ONCE(!mm)) - return -EPERM; - - if (!npages) - return 0; - - down_write(&mm->mmap_sem); - locked = mm->locked_vm + npages; - lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked > lock_limit && !capable(CAP_IPC_LOCK)) - ret = -ENOMEM; - else - mm->locked_vm += npages; - - pr_debug("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n", current->pid, - npages << PAGE_SHIFT, - mm->locked_vm << PAGE_SHIFT, - rlimit(RLIMIT_MEMLOCK), - ret ? " - exceeded" : ""); - - up_write(&mm->mmap_sem); - - return ret; -} - -static void decrement_locked_vm(struct mm_struct *mm, long npages) -{ - if (!mm || !npages) - return; - - down_write(&mm->mmap_sem); - if (WARN_ON_ONCE(npages > mm->locked_vm)) - npages = mm->locked_vm; - mm->locked_vm -= npages; - pr_debug("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n", current->pid, - npages << PAGE_SHIFT, - mm->locked_vm << PAGE_SHIFT, - rlimit(RLIMIT_MEMLOCK)); - up_write(&mm->mmap_sem); -} - /* * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation * @@ -333,7 +289,7 @@ static int tce_iommu_enable(struct tce_container *container) return ret; locked = table_group->tce32_size >> PAGE_SHIFT; - ret = try_increment_locked_vm(container->mm, locked); + ret = account_locked_vm(container->mm, locked, true); if (ret) return ret; @@ -352,7 +308,7 @@ static void tce_iommu_disable(struct tce_container *container) container->enabled = false; BUG_ON(!container->mm); - decrement_locked_vm(container->mm, container->locked_pages); + account_locked_vm(container->mm, container->locked_pages, false); } static void *tce_iommu_open(unsigned long arg) @@ -656,7 +612,7 @@ static long tce_iommu_create_table(struct tce_container *container, if (!table_size) return -EINVAL; - ret = try_increment_locked_vm(container->mm, table_size >> PAGE_SHIFT); + ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true); if (ret) return ret; @@ -675,7 +631,7 @@ static void tce_iommu_free_table(struct tce_container *container, unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT; iommu_tce_table_put(tbl); - decrement_locked_vm(container->mm, pages); + account_locked_vm(container->mm, pages, false); } static long tce_iommu_create_window(struct tce_container *container, diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index add34adfadc7..054391f30fa8 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -272,21 +272,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) ret = down_write_killable(&mm->mmap_sem); if (!ret) { - if (npage > 0) { - if (!dma->lock_cap) { - unsigned long limit; - - limit = task_rlimit(dma->task, - RLIMIT_MEMLOCK) >> PAGE_SHIFT; - - if (mm->locked_vm + npage > limit) - ret = -ENOMEM; - } - } - - if (!ret) - mm->locked_vm += npage; - + ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task, + dma->lock_cap); up_write(&mm->mmap_sem); } diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 247e5585af5d..1a2dd53caade 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -956,7 +956,7 @@ static void handle_tx(struct vhost_net *net) if (!sock) goto out; - if (!vq_iotlb_prefetch(vq)) + if (!vq_meta_prefetch(vq)) goto out; vhost_disable_notify(&net->dev, vq); @@ -1125,7 +1125,7 @@ static void handle_rx(struct vhost_net *net) if (!sock) goto out; - if (!vq_iotlb_prefetch(vq)) + if (!vq_meta_prefetch(vq)) goto out; vhost_disable_notify(&net->dev, vq); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index ff8892c38666..0536f8526359 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -298,6 +298,160 @@ static void vhost_vq_meta_reset(struct vhost_dev *d) __vhost_vq_meta_reset(d->vqs[i]); } +#if VHOST_ARCH_CAN_ACCEL_UACCESS +static void vhost_map_unprefetch(struct vhost_map *map) +{ + kfree(map->pages); + map->pages = NULL; + map->npages = 0; + map->addr = NULL; +} + +static void vhost_uninit_vq_maps(struct vhost_virtqueue *vq) +{ + struct vhost_map *map[VHOST_NUM_ADDRS]; + int i; + + spin_lock(&vq->mmu_lock); + for (i = 0; i < VHOST_NUM_ADDRS; i++) { + map[i] = rcu_dereference_protected(vq->maps[i], + lockdep_is_held(&vq->mmu_lock)); + if (map[i]) + rcu_assign_pointer(vq->maps[i], NULL); + } + spin_unlock(&vq->mmu_lock); + + synchronize_rcu(); + + for (i = 0; i < VHOST_NUM_ADDRS; i++) + if (map[i]) + vhost_map_unprefetch(map[i]); + +} + +static void vhost_reset_vq_maps(struct vhost_virtqueue *vq) +{ + int i; + + vhost_uninit_vq_maps(vq); + for (i = 0; i < VHOST_NUM_ADDRS; i++) + vq->uaddrs[i].size = 0; +} + +static bool vhost_map_range_overlap(struct vhost_uaddr *uaddr, + unsigned long start, + unsigned long end) +{ + if (unlikely(!uaddr->size)) + return false; + + return !(end < uaddr->uaddr || start > uaddr->uaddr - 1 + uaddr->size); +} + +static void vhost_invalidate_vq_start(struct vhost_virtqueue *vq, + int index, + unsigned long start, + unsigned long end) +{ + struct vhost_uaddr *uaddr = &vq->uaddrs[index]; + struct vhost_map *map; + int i; + + if (!vhost_map_range_overlap(uaddr, start, end)) + return; + + spin_lock(&vq->mmu_lock); + ++vq->invalidate_count; + + map = rcu_dereference_protected(vq->maps[index], + lockdep_is_held(&vq->mmu_lock)); + if (map) { + if (uaddr->write) { + for (i = 0; i < map->npages; i++) + set_page_dirty(map->pages[i]); + } + rcu_assign_pointer(vq->maps[index], NULL); + } + spin_unlock(&vq->mmu_lock); + + if (map) { + synchronize_rcu(); + vhost_map_unprefetch(map); + } +} + +static void vhost_invalidate_vq_end(struct vhost_virtqueue *vq, + int index, + unsigned long start, + unsigned long end) +{ + if (!vhost_map_range_overlap(&vq->uaddrs[index], start, end)) + return; + + spin_lock(&vq->mmu_lock); + --vq->invalidate_count; + spin_unlock(&vq->mmu_lock); +} + +static int vhost_invalidate_range_start(struct mmu_notifier *mn, + const struct mmu_notifier_range *range) +{ + struct vhost_dev *dev = container_of(mn, struct vhost_dev, + mmu_notifier); + int i, j; + + if (!mmu_notifier_range_blockable(range)) + return -EAGAIN; + + for (i = 0; i < dev->nvqs; i++) { + struct vhost_virtqueue *vq = dev->vqs[i]; + + for (j = 0; j < VHOST_NUM_ADDRS; j++) + vhost_invalidate_vq_start(vq, j, + range->start, + range->end); + } + + return 0; +} + +static void vhost_invalidate_range_end(struct mmu_notifier *mn, + const struct mmu_notifier_range *range) +{ + struct vhost_dev *dev = container_of(mn, struct vhost_dev, + mmu_notifier); + int i, j; + + for (i = 0; i < dev->nvqs; i++) { + struct vhost_virtqueue *vq = dev->vqs[i]; + + for (j = 0; j < VHOST_NUM_ADDRS; j++) + vhost_invalidate_vq_end(vq, j, + range->start, + range->end); + } +} + +static const struct mmu_notifier_ops vhost_mmu_notifier_ops = { + .invalidate_range_start = vhost_invalidate_range_start, + .invalidate_range_end = vhost_invalidate_range_end, +}; + +static void vhost_init_maps(struct vhost_dev *dev) +{ + struct vhost_virtqueue *vq; + int i, j; + + dev->mmu_notifier.ops = &vhost_mmu_notifier_ops; + + for (i = 0; i < dev->nvqs; ++i) { + vq = dev->vqs[i]; + for (j = 0; j < VHOST_NUM_ADDRS; j++) + RCU_INIT_POINTER(vq->maps[j], NULL); + } +} +#endif + static void vhost_vq_reset(struct vhost_dev *dev, struct vhost_virtqueue *vq) { @@ -326,7 +480,11 @@ static void vhost_vq_reset(struct vhost_dev *dev, vq->busyloop_timeout = 0; vq->umem = NULL; vq->iotlb = NULL; + vq->invalidate_count = 0; __vhost_vq_meta_reset(vq); +#if VHOST_ARCH_CAN_ACCEL_UACCESS + vhost_reset_vq_maps(vq); +#endif } static int vhost_worker(void *data) @@ -427,6 +585,32 @@ bool vhost_exceeds_weight(struct vhost_virtqueue *vq, } EXPORT_SYMBOL_GPL(vhost_exceeds_weight); +static size_t vhost_get_avail_size(struct vhost_virtqueue *vq, + unsigned int num) +{ + size_t event __maybe_unused = + vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; + + return sizeof(*vq->avail) + + sizeof(*vq->avail->ring) * num + event; +} + +static size_t vhost_get_used_size(struct vhost_virtqueue *vq, + unsigned int num) +{ + size_t event __maybe_unused = + vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; + + return sizeof(*vq->used) + + sizeof(*vq->used->ring) * num + event; +} + +static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, + unsigned int num) +{ + return sizeof(*vq->desc) * num; +} + void vhost_dev_init(struct vhost_dev *dev, struct vhost_virtqueue **vqs, int nvqs, int iov_limit, int weight, int byte_weight) @@ -450,7 +634,9 @@ void vhost_dev_init(struct vhost_dev *dev, INIT_LIST_HEAD(&dev->read_list); INIT_LIST_HEAD(&dev->pending_list); spin_lock_init(&dev->iotlb_lock); - +#if VHOST_ARCH_CAN_ACCEL_UACCESS + vhost_init_maps(dev); +#endif for (i = 0; i < dev->nvqs; ++i) { vq = dev->vqs[i]; @@ -459,6 +645,7 @@ void vhost_dev_init(struct vhost_dev *dev, vq->heads = NULL; vq->dev = dev; mutex_init(&vq->mutex); + spin_lock_init(&vq->mmu_lock); vhost_vq_reset(dev, vq); if (vq->handle_kick) vhost_poll_init(&vq->poll, vq->handle_kick, @@ -538,7 +725,18 @@ long vhost_dev_set_owner(struct vhost_dev *dev) if (err) goto err_cgroup; +#if VHOST_ARCH_CAN_ACCEL_UACCESS + err = mmu_notifier_register(&dev->mmu_notifier, dev->mm); + if (err) + goto err_mmu_notifier; +#endif + return 0; + +#if VHOST_ARCH_CAN_ACCEL_UACCESS +err_mmu_notifier: + vhost_dev_free_iovecs(dev); +#endif err_cgroup: kthread_stop(worker); dev->worker = NULL; @@ -629,6 +827,107 @@ static void vhost_clear_msg(struct vhost_dev *dev) spin_unlock(&dev->iotlb_lock); } +#if VHOST_ARCH_CAN_ACCEL_UACCESS +static void vhost_setup_uaddr(struct vhost_virtqueue *vq, + int index, unsigned long uaddr, + size_t size, bool write) +{ + struct vhost_uaddr *addr = &vq->uaddrs[index]; + + addr->uaddr = uaddr; + addr->size = size; + addr->write = write; +} + +static void vhost_setup_vq_uaddr(struct vhost_virtqueue *vq) +{ + vhost_setup_uaddr(vq, VHOST_ADDR_DESC, + (unsigned long)vq->desc, + vhost_get_desc_size(vq, vq->num), + false); + vhost_setup_uaddr(vq, VHOST_ADDR_AVAIL, + (unsigned long)vq->avail, + vhost_get_avail_size(vq, vq->num), + false); + vhost_setup_uaddr(vq, VHOST_ADDR_USED, + (unsigned long)vq->used, + vhost_get_used_size(vq, vq->num), + true); +} + +static int vhost_map_prefetch(struct vhost_virtqueue *vq, + int index) +{ + struct vhost_map *map; + struct vhost_uaddr *uaddr = &vq->uaddrs[index]; + struct page **pages; + int npages = DIV_ROUND_UP(uaddr->size, PAGE_SIZE); + int npinned; + void *vaddr, *v; + int err; + int i; + + spin_lock(&vq->mmu_lock); + + err = -EFAULT; + if (vq->invalidate_count) + goto err; + + err = -ENOMEM; + map = kmalloc(sizeof(*map), GFP_ATOMIC); + if (!map) + goto err; + + pages = kmalloc_array(npages, sizeof(struct page *), GFP_ATOMIC); + if (!pages) + goto err_pages; + + err = EFAULT; + npinned = __get_user_pages_fast(uaddr->uaddr, npages, + uaddr->write, pages); + if (npinned > 0) + release_pages(pages, npinned); + if (npinned != npages) + goto err_gup; + + for (i = 0; i < npinned; i++) + if (PageHighMem(pages[i])) + goto err_gup; + + vaddr = v = page_address(pages[0]); + + /* For simplicity, fallback to userspace address if VA is not + * contigious. + */ + for (i = 1; i < npinned; i++) { + v += PAGE_SIZE; + if (v != page_address(pages[i])) + goto err_gup; + } + + map->addr = vaddr + (uaddr->uaddr & (PAGE_SIZE - 1)); + map->npages = npages; + map->pages = pages; + + rcu_assign_pointer(vq->maps[index], map); + /* No need for a synchronize_rcu(). This function should be + * called by dev->worker so we are serialized with all + * readers. + */ + spin_unlock(&vq->mmu_lock); + + return 0; + +err_gup: + kfree(pages); +err_pages: + kfree(map); +err: + spin_unlock(&vq->mmu_lock); + return err; +} +#endif + void vhost_dev_cleanup(struct vhost_dev *dev) { int i; @@ -658,8 +957,16 @@ void vhost_dev_cleanup(struct vhost_dev *dev) kthread_stop(dev->worker); dev->worker = NULL; } - if (dev->mm) + if (dev->mm) { +#if VHOST_ARCH_CAN_ACCEL_UACCESS + mmu_notifier_unregister(&dev->mmu_notifier, dev->mm); +#endif mmput(dev->mm); + } +#if VHOST_ARCH_CAN_ACCEL_UACCESS + for (i = 0; i < dev->nvqs; i++) + vhost_uninit_vq_maps(dev->vqs[i]); +#endif dev->mm = NULL; } EXPORT_SYMBOL_GPL(vhost_dev_cleanup); @@ -886,6 +1193,113 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, ret; \ }) +static inline int vhost_put_avail_event(struct vhost_virtqueue *vq) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_used *used; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); + if (likely(map)) { + used = map->addr; + *((__virtio16 *)&used->ring[vq->num]) = + cpu_to_vhost16(vq, vq->avail_idx); + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), + vhost_avail_event(vq)); +} + +static inline int vhost_put_used(struct vhost_virtqueue *vq, + struct vring_used_elem *head, int idx, + int count) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_used *used; + size_t size; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); + if (likely(map)) { + used = map->addr; + size = count * sizeof(*head); + memcpy(used->ring + idx, head, size); + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_copy_to_user(vq, vq->used->ring + idx, head, + count * sizeof(*head)); +} + +static inline int vhost_put_used_flags(struct vhost_virtqueue *vq) + +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_used *used; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); + if (likely(map)) { + used = map->addr; + used->flags = cpu_to_vhost16(vq, vq->used_flags); + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), + &vq->used->flags); +} + +static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) + +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_used *used; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); + if (likely(map)) { + used = map->addr; + used->idx = cpu_to_vhost16(vq, vq->last_used_idx); + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), + &vq->used->idx); +} + #define vhost_get_user(vq, x, ptr, type) \ ({ \ int ret; \ @@ -924,6 +1338,155 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d) mutex_unlock(&d->vqs[i]->mutex); } +static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, + __virtio16 *idx) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_avail *avail; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); + if (likely(map)) { + avail = map->addr; + *idx = avail->idx; + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_get_avail(vq, *idx, &vq->avail->idx); +} + +static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, + __virtio16 *head, int idx) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_avail *avail; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); + if (likely(map)) { + avail = map->addr; + *head = avail->ring[idx & (vq->num - 1)]; + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_get_avail(vq, *head, + &vq->avail->ring[idx & (vq->num - 1)]); +} + +static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq, + __virtio16 *flags) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_avail *avail; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); + if (likely(map)) { + avail = map->addr; + *flags = avail->flags; + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_get_avail(vq, *flags, &vq->avail->flags); +} + +static inline int vhost_get_used_event(struct vhost_virtqueue *vq, + __virtio16 *event) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_avail *avail; + + if (!vq->iotlb) { + rcu_read_lock(); + map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); + if (likely(map)) { + avail = map->addr; + *event = (__virtio16)avail->ring[vq->num]; + rcu_read_unlock(); + return 0; + } + rcu_read_unlock(); + } +#endif + + return vhost_get_avail(vq, *event, vhost_used_event(vq)); +} + +static inline int vhost_get_used_idx(struct vhost_virtqueue *vq, + __virtio16 *idx) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_used *used; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); + if (likely(map)) { + used = map->addr; + *idx = used->idx; + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_get_used(vq, *idx, &vq->used->idx); +} + +static inline int vhost_get_desc(struct vhost_virtqueue *vq, + struct vring_desc *desc, int idx) +{ +#if VHOST_ARCH_CAN_ACCEL_UACCESS + struct vhost_map *map; + struct vring_desc *d; + + if (!vq->iotlb) { + rcu_read_lock(); + + map = rcu_dereference(vq->maps[VHOST_ADDR_DESC]); + if (likely(map)) { + d = map->addr; + *desc = *(d + idx); + rcu_read_unlock(); + return 0; + } + + rcu_read_unlock(); + } +#endif + + return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc)); +} + static int vhost_new_umem_range(struct vhost_umem *umem, u64 start, u64 size, u64 end, u64 userspace_addr, int perm) @@ -1209,13 +1772,9 @@ static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, struct vring_used __user *used) { - size_t s __maybe_unused = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; - - return access_ok(desc, num * sizeof *desc) && - access_ok(avail, - sizeof *avail + num * sizeof *avail->ring + s) && - access_ok(used, - sizeof *used + num * sizeof *used->ring + s); + return access_ok(desc, vhost_get_desc_size(vq, num)) && + access_ok(avail, vhost_get_avail_size(vq, num)) && + access_ok(used, vhost_get_used_size(vq, num)); } static void vhost_vq_meta_update(struct vhost_virtqueue *vq, @@ -1265,26 +1824,42 @@ static bool iotlb_access_ok(struct vhost_virtqueue *vq, return true; } -int vq_iotlb_prefetch(struct vhost_virtqueue *vq) +#if VHOST_ARCH_CAN_ACCEL_UACCESS +static void vhost_vq_map_prefetch(struct vhost_virtqueue *vq) +{ + struct vhost_map __rcu *map; + int i; + + for (i = 0; i < VHOST_NUM_ADDRS; i++) { + rcu_read_lock(); + map = rcu_dereference(vq->maps[i]); + rcu_read_unlock(); + if (unlikely(!map)) + vhost_map_prefetch(vq, i); + } +} +#endif + +int vq_meta_prefetch(struct vhost_virtqueue *vq) { - size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; unsigned int num = vq->num; - if (!vq->iotlb) + if (!vq->iotlb) { +#if VHOST_ARCH_CAN_ACCEL_UACCESS + vhost_vq_map_prefetch(vq); +#endif return 1; + } return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, - num * sizeof(*vq->desc), VHOST_ADDR_DESC) && + vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail, - sizeof *vq->avail + - num * sizeof(*vq->avail->ring) + s, + vhost_get_avail_size(vq, num), VHOST_ADDR_AVAIL) && iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used, - sizeof *vq->used + - num * sizeof(*vq->used->ring) + s, - VHOST_ADDR_USED); + vhost_get_used_size(vq, num), VHOST_ADDR_USED); } -EXPORT_SYMBOL_GPL(vq_iotlb_prefetch); +EXPORT_SYMBOL_GPL(vq_meta_prefetch); /* Can we log writes? */ /* Caller should have device mutex but not vq mutex */ @@ -1299,13 +1874,10 @@ EXPORT_SYMBOL_GPL(vhost_log_access_ok); static bool vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base) { - size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; - return vq_memory_access_ok(log_base, vq->umem, vhost_has_feature(vq, VHOST_F_LOG_ALL)) && (!vq->log_used || log_access_ok(log_base, vq->log_addr, - sizeof *vq->used + - vq->num * sizeof *vq->used->ring + s)); + vhost_get_used_size(vq, vq->num))); } /* Can we start vq? */ @@ -1405,6 +1977,121 @@ err: return -EFAULT; } +static long vhost_vring_set_num(struct vhost_dev *d, + struct vhost_virtqueue *vq, + void __user *argp) +{ + struct vhost_vring_state s; + + /* Resizing ring with an active backend? + * You don't want to do that. */ + if (vq->private_data) + return -EBUSY; + + if (copy_from_user(&s, argp, sizeof s)) + return -EFAULT; + + if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) + return -EINVAL; + vq->num = s.num; + + return 0; +} + +static long vhost_vring_set_addr(struct vhost_dev *d, + struct vhost_virtqueue *vq, + void __user *argp) +{ + struct vhost_vring_addr a; + + if (copy_from_user(&a, argp, sizeof a)) + return -EFAULT; + if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) + return -EOPNOTSUPP; + + /* For 32bit, verify that the top 32bits of the user + data are set to zero. */ + if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr || + (u64)(unsigned long)a.used_user_addr != a.used_user_addr || + (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) + return -EFAULT; + + /* Make sure it's safe to cast pointers to vring types. */ + BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); + BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); + if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) || + (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) || + (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) + return -EINVAL; + + /* We only verify access here if backend is configured. + * If it is not, we don't as size might not have been setup. + * We will verify when backend is configured. */ + if (vq->private_data) { + if (!vq_access_ok(vq, vq->num, + (void __user *)(unsigned long)a.desc_user_addr, + (void __user *)(unsigned long)a.avail_user_addr, + (void __user *)(unsigned long)a.used_user_addr)) + return -EINVAL; + + /* Also validate log access for used ring if enabled. */ + if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) && + !log_access_ok(vq->log_base, a.log_guest_addr, + sizeof *vq->used + + vq->num * sizeof *vq->used->ring)) + return -EINVAL; + } + + vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); + vq->desc = (void __user *)(unsigned long)a.desc_user_addr; + vq->avail = (void __user *)(unsigned long)a.avail_user_addr; + vq->log_addr = a.log_guest_addr; + vq->used = (void __user *)(unsigned long)a.used_user_addr; + + return 0; +} + +static long vhost_vring_set_num_addr(struct vhost_dev *d, + struct vhost_virtqueue *vq, + unsigned int ioctl, + void __user *argp) +{ + long r; + + mutex_lock(&vq->mutex); + +#if VHOST_ARCH_CAN_ACCEL_UACCESS + /* Unregister MMU notifer to allow invalidation callback + * can access vq->uaddrs[] without holding a lock. + */ + if (d->mm) + mmu_notifier_unregister(&d->mmu_notifier, d->mm); + + vhost_uninit_vq_maps(vq); +#endif + + switch (ioctl) { + case VHOST_SET_VRING_NUM: + r = vhost_vring_set_num(d, vq, argp); + break; + case VHOST_SET_VRING_ADDR: + r = vhost_vring_set_addr(d, vq, argp); + break; + default: + BUG(); + } + +#if VHOST_ARCH_CAN_ACCEL_UACCESS + vhost_setup_vq_uaddr(vq); + + if (d->mm) + mmu_notifier_register(&d->mmu_notifier, d->mm); +#endif + + mutex_unlock(&vq->mutex); + + return r; +} long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) { struct file *eventfp, *filep = NULL; @@ -1414,7 +2101,6 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg struct vhost_virtqueue *vq; struct vhost_vring_state s; struct vhost_vring_file f; - struct vhost_vring_addr a; u32 idx; long r; @@ -1427,26 +2113,14 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg idx = array_index_nospec(idx, d->nvqs); vq = d->vqs[idx]; + if (ioctl == VHOST_SET_VRING_NUM || + ioctl == VHOST_SET_VRING_ADDR) { + return vhost_vring_set_num_addr(d, vq, ioctl, argp); + } + mutex_lock(&vq->mutex); switch (ioctl) { - case VHOST_SET_VRING_NUM: - /* Resizing ring with an active backend? - * You don't want to do that. */ - if (vq->private_data) { - r = -EBUSY; - break; - } - if (copy_from_user(&s, argp, sizeof s)) { - r = -EFAULT; - break; - } - if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) { - r = -EINVAL; - break; - } - vq->num = s.num; - break; case VHOST_SET_VRING_BASE: /* Moving base with an active backend? * You don't want to do that. */ @@ -1472,62 +2146,6 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg if (copy_to_user(argp, &s, sizeof s)) r = -EFAULT; break; - case VHOST_SET_VRING_ADDR: - if (copy_from_user(&a, argp, sizeof a)) { - r = -EFAULT; - break; - } - if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) { - r = -EOPNOTSUPP; - break; - } - /* For 32bit, verify that the top 32bits of the user - data are set to zero. */ - if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr || - (u64)(unsigned long)a.used_user_addr != a.used_user_addr || - (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) { - r = -EFAULT; - break; - } - - /* Make sure it's safe to cast pointers to vring types. */ - BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); - BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); - if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) || - (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) || - (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) { - r = -EINVAL; - break; - } - - /* We only verify access here if backend is configured. - * If it is not, we don't as size might not have been setup. - * We will verify when backend is configured. */ - if (vq->private_data) { - if (!vq_access_ok(vq, vq->num, - (void __user *)(unsigned long)a.desc_user_addr, - (void __user *)(unsigned long)a.avail_user_addr, - (void __user *)(unsigned long)a.used_user_addr)) { - r = -EINVAL; - break; - } - - /* Also validate log access for used ring if enabled. */ - if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) && - !log_access_ok(vq->log_base, a.log_guest_addr, - sizeof *vq->used + - vq->num * sizeof *vq->used->ring)) { - r = -EINVAL; - break; - } - } - - vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); - vq->desc = (void __user *)(unsigned long)a.desc_user_addr; - vq->avail = (void __user *)(unsigned long)a.avail_user_addr; - vq->log_addr = a.log_guest_addr; - vq->used = (void __user *)(unsigned long)a.used_user_addr; - break; case VHOST_SET_VRING_KICK: if (copy_from_user(&f, argp, sizeof f)) { r = -EFAULT; @@ -1861,8 +2479,7 @@ EXPORT_SYMBOL_GPL(vhost_log_write); static int vhost_update_used_flags(struct vhost_virtqueue *vq) { void __user *used; - if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), - &vq->used->flags) < 0) + if (vhost_put_used_flags(vq)) return -EFAULT; if (unlikely(vq->log_used)) { /* Make sure the flag is seen before log. */ @@ -1879,8 +2496,7 @@ static int vhost_update_used_flags(struct vhost_virtqueue *vq) static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) { - if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), - vhost_avail_event(vq))) + if (vhost_put_avail_event(vq)) return -EFAULT; if (unlikely(vq->log_used)) { void __user *used; @@ -1916,7 +2532,7 @@ int vhost_vq_init_access(struct vhost_virtqueue *vq) r = -EFAULT; goto err; } - r = vhost_get_used(vq, last_used_idx, &vq->used->idx); + r = vhost_get_used_idx(vq, &last_used_idx); if (r) { vq_err(vq, "Can't access used idx at %p\n", &vq->used->idx); @@ -2115,7 +2731,7 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq, last_avail_idx = vq->last_avail_idx; if (vq->avail_idx == vq->last_avail_idx) { - if (unlikely(vhost_get_avail(vq, avail_idx, &vq->avail->idx))) { + if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) { vq_err(vq, "Failed to access avail idx at %p\n", &vq->avail->idx); return -EFAULT; @@ -2142,8 +2758,7 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq, /* Grab the next descriptor number they're advertising, and increment * the index we've seen. */ - if (unlikely(vhost_get_avail(vq, ring_head, - &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) { + if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) { vq_err(vq, "Failed to read head: idx %d address %p\n", last_avail_idx, &vq->avail->ring[last_avail_idx % vq->num]); @@ -2178,8 +2793,7 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq, i, vq->num, head); return -EINVAL; } - ret = vhost_copy_from_user(vq, &desc, vq->desc + i, - sizeof desc); + ret = vhost_get_desc(vq, &desc, i); if (unlikely(ret)) { vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", i, vq->desc + i); @@ -2272,16 +2886,7 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq, start = vq->last_used_idx & (vq->num - 1); used = vq->used->ring + start; - if (count == 1) { - if (vhost_put_user(vq, heads[0].id, &used->id)) { - vq_err(vq, "Failed to write used id"); - return -EFAULT; - } - if (vhost_put_user(vq, heads[0].len, &used->len)) { - vq_err(vq, "Failed to write used len"); - return -EFAULT; - } - } else if (vhost_copy_to_user(vq, used, heads, count * sizeof *used)) { + if (vhost_put_used(vq, heads, start, count)) { vq_err(vq, "Failed to write used"); return -EFAULT; } @@ -2323,8 +2928,7 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, /* Make sure buffer is written before we update index. */ smp_wmb(); - if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), - &vq->used->idx)) { + if (vhost_put_used_idx(vq)) { vq_err(vq, "Failed to increment used idx"); return -EFAULT; } @@ -2357,7 +2961,7 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { __virtio16 flags; - if (vhost_get_avail(vq, flags, &vq->avail->flags)) { + if (vhost_get_avail_flags(vq, &flags)) { vq_err(vq, "Failed to get flags"); return true; } @@ -2371,7 +2975,7 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) if (unlikely(!v)) return true; - if (vhost_get_avail(vq, event, vhost_used_event(vq))) { + if (vhost_get_used_event(vq, &event)) { vq_err(vq, "Failed to get used event idx"); return true; } @@ -2416,7 +3020,7 @@ bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) if (vq->avail_idx != vq->last_avail_idx) return false; - r = vhost_get_avail(vq, avail_idx, &vq->avail->idx); + r = vhost_get_avail_idx(vq, &avail_idx); if (unlikely(r)) return false; vq->avail_idx = vhost16_to_cpu(vq, avail_idx); @@ -2452,7 +3056,7 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) /* They could have slipped one in as we were doing that: make * sure it's written, then check again. */ smp_mb(); - r = vhost_get_avail(vq, avail_idx, &vq->avail->idx); + r = vhost_get_avail_idx(vq, &avail_idx); if (r) { vq_err(vq, "Failed to check avail idx at %p: %d\n", &vq->avail->idx, r); diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 27a78a9b8cc7..819296332913 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -12,6 +12,9 @@ #include <linux/virtio_config.h> #include <linux/virtio_ring.h> #include <linux/atomic.h> +#include <linux/pagemap.h> +#include <linux/mmu_notifier.h> +#include <asm/cacheflush.h> struct vhost_work; typedef void (*vhost_work_fn_t)(struct vhost_work *work); @@ -80,6 +83,24 @@ enum vhost_uaddr_type { VHOST_NUM_ADDRS = 3, }; +struct vhost_map { + int npages; + void *addr; + struct page **pages; +}; + +struct vhost_uaddr { + unsigned long uaddr; + size_t size; + bool write; +}; + +#if defined(CONFIG_MMU_NOTIFIER) && ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 0 +#define VHOST_ARCH_CAN_ACCEL_UACCESS 1 +#else +#define VHOST_ARCH_CAN_ACCEL_UACCESS 0 +#endif + /* The virtqueue structure describes a queue attached to a device. */ struct vhost_virtqueue { struct vhost_dev *dev; @@ -90,7 +111,22 @@ struct vhost_virtqueue { struct vring_desc __user *desc; struct vring_avail __user *avail; struct vring_used __user *used; + +#if VHOST_ARCH_CAN_ACCEL_UACCESS + /* Read by memory accessors, modified by meta data + * prefetching, MMU notifier and vring ioctl(). + * Synchonrized through mmu_lock (writers) and RCU (writers + * and readers). + */ + struct vhost_map __rcu *maps[VHOST_NUM_ADDRS]; + /* Read by MMU notifier, modified by vring ioctl(), + * synchronized through MMU notifier + * registering/unregistering. + */ + struct vhost_uaddr uaddrs[VHOST_NUM_ADDRS]; +#endif const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS]; + struct file *kick; struct eventfd_ctx *call_ctx; struct eventfd_ctx *error_ctx; @@ -145,6 +181,8 @@ struct vhost_virtqueue { bool user_be; #endif u32 busyloop_timeout; + spinlock_t mmu_lock; + int invalidate_count; }; struct vhost_msg_node { @@ -158,6 +196,9 @@ struct vhost_msg_node { struct vhost_dev { struct mm_struct *mm; +#ifdef CONFIG_MMU_NOTIFIER + struct mmu_notifier mmu_notifier; +#endif struct mutex mutex; struct vhost_virtqueue **vqs; int nvqs; @@ -212,7 +253,7 @@ bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *); int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, unsigned int log_num, u64 len, struct iovec *iov, int count); -int vq_iotlb_prefetch(struct vhost_virtqueue *vq); +int vq_meta_prefetch(struct vhost_virtqueue *vq); struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type); void vhost_enqueue_msg(struct vhost_dev *dev, diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 023fc3bc01c6..078615cf2afc 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -43,6 +43,17 @@ config VIRTIO_PCI_LEGACY If unsure, say Y. +config VIRTIO_PMEM + tristate "Support for virtio pmem driver" + depends on VIRTIO + depends on LIBNVDIMM + help + This driver provides access to virtio-pmem devices, storage devices + that are mapped into the physical address space - similar to NVDIMMs + - with a virtio-based flushing interface. + + If unsure, say Y. + config VIRTIO_BALLOON tristate "Virtio balloon driver" depends on VIRTIO diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index f363fbeb5ab0..e09edb5c5e06 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -463,9 +463,14 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct irq_affinity *desc) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - unsigned int irq = platform_get_irq(vm_dev->pdev, 0); + int irq = platform_get_irq(vm_dev->pdev, 0); int i, err, queue_idx = 0; + if (irq < 0) { + dev_err(&vdev->dev, "Cannot get IRQ resource\n"); + return irq; + } + err = request_irq(irq, vm_interrupt, IRQF_SHARED, dev_name(&vdev->dev), vm_dev); if (err) diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 6cad0b33d7ad..8188963a405b 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -58,6 +58,15 @@ config WATCHDOG_HANDLE_BOOT_ENABLED the watchdog on its own. Thus if your userspace does not start fast enough your device will reboot. +config WATCHDOG_OPEN_TIMEOUT + int "Timeout value for opening watchdog device" + default 0 + help + The maximum time, in seconds, for which the watchdog framework takes + care of pinging a hardware watchdog. A value of 0 means infinite. The + value set here can be overridden by the commandline parameter + "watchdog.open_timeout". + config WATCHDOG_SYSFS bool "Read different watchdog information through sysfs" help @@ -717,6 +726,7 @@ config IMX2_WDT config IMX_SC_WDT tristate "IMX SC Watchdog" depends on HAVE_ARM_SMCCC + depends on IMX_SCU select WATCHDOG_CORE help This is the driver for the system controller watchdog diff --git a/drivers/watchdog/acquirewdt.c b/drivers/watchdog/acquirewdt.c index 957d1255d4ca..848db958411e 100644 --- a/drivers/watchdog/acquirewdt.c +++ b/drivers/watchdog/acquirewdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Acquire Single Board Computer Watchdog Timer driver * @@ -6,11 +7,6 @@ * (c) Copyright 1996 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/advantechwdt.c b/drivers/watchdog/advantechwdt.c index 2766af292a71..0d02bb275b3d 100644 --- a/drivers/watchdog/advantechwdt.c +++ b/drivers/watchdog/advantechwdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Advantech Single Board Computer WDT driver * @@ -9,11 +10,6 @@ * (c) Copyright 1996 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/aspeed_wdt.c b/drivers/watchdog/aspeed_wdt.c index f0148637e5dd..cc71861e033a 100644 --- a/drivers/watchdog/aspeed_wdt.c +++ b/drivers/watchdog/aspeed_wdt.c @@ -309,13 +309,7 @@ static int aspeed_wdt_probe(struct platform_device *pdev) if (status & WDT_TIMEOUT_STATUS_BOOT_SECONDARY) wdt->wdd.bootstatus = WDIOF_CARDRESET; - ret = devm_watchdog_register_device(dev, &wdt->wdd); - if (ret) { - dev_err(dev, "failed to register\n"); - return ret; - } - - return 0; + return devm_watchdog_register_device(dev, &wdt->wdd); } static struct platform_driver aspeed_watchdog_driver = { diff --git a/drivers/watchdog/bcm2835_wdt.c b/drivers/watchdog/bcm2835_wdt.c index 560c1c54c177..dec6ca019bea 100644 --- a/drivers/watchdog/bcm2835_wdt.c +++ b/drivers/watchdog/bcm2835_wdt.c @@ -202,10 +202,8 @@ static int bcm2835_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(&bcm2835_wdt_wdd); err = devm_watchdog_register_device(dev, &bcm2835_wdt_wdd); - if (err) { - dev_err(dev, "Failed to register watchdog device"); + if (err) return err; - } if (pm_power_off == NULL) { pm_power_off = bcm2835_power_off; @@ -240,6 +238,7 @@ module_param(nowayout, bool, 0); MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); +MODULE_ALIAS("platform:bcm2835-wdt"); MODULE_AUTHOR("Lubomir Rintel <lkundrak@v3.sk>"); MODULE_DESCRIPTION("Driver for Broadcom BCM2835 watchdog timer"); MODULE_LICENSE("GPL"); diff --git a/drivers/watchdog/bcm7038_wdt.c b/drivers/watchdog/bcm7038_wdt.c index d3d88f6703d7..979caa18d3c8 100644 --- a/drivers/watchdog/bcm7038_wdt.c +++ b/drivers/watchdog/bcm7038_wdt.c @@ -159,10 +159,8 @@ static int bcm7038_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(&wdt->wdd); watchdog_stop_on_unregister(&wdt->wdd); err = devm_watchdog_register_device(dev, &wdt->wdd); - if (err) { - dev_err(dev, "Failed to register watchdog device\n"); + if (err) return err; - } dev_info(dev, "Registered BCM7038 Watchdog\n"); diff --git a/drivers/watchdog/bcm_kona_wdt.c b/drivers/watchdog/bcm_kona_wdt.c index 921291025680..eb850a8d19df 100644 --- a/drivers/watchdog/bcm_kona_wdt.c +++ b/drivers/watchdog/bcm_kona_wdt.c @@ -301,10 +301,8 @@ static int bcm_kona_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(&bcm_kona_wdt_wdd); watchdog_stop_on_unregister(&bcm_kona_wdt_wdd); ret = devm_watchdog_register_device(dev, &bcm_kona_wdt_wdd); - if (ret) { - dev_err(dev, "Failed to register watchdog device"); + if (ret) return ret; - } bcm_kona_wdt_debug_init(pdev); dev_dbg(dev, "Broadcom Kona Watchdog Timer"); diff --git a/drivers/watchdog/cadence_wdt.c b/drivers/watchdog/cadence_wdt.c index a22f2d431a35..f8d4e91d0383 100644 --- a/drivers/watchdog/cadence_wdt.c +++ b/drivers/watchdog/cadence_wdt.c @@ -363,10 +363,8 @@ static int cdns_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(cdns_wdt_device); watchdog_stop_on_unregister(cdns_wdt_device); ret = devm_watchdog_register_device(dev, cdns_wdt_device); - if (ret) { - dev_err(dev, "Failed to register wdt device\n"); + if (ret) return ret; - } platform_set_drvdata(pdev, wdt); dev_info(dev, "Xilinx Watchdog Timer at %p with timeout %ds%s\n", diff --git a/drivers/watchdog/da9052_wdt.c b/drivers/watchdog/da9052_wdt.c index a2feef1ff307..d708c091bf1b 100644 --- a/drivers/watchdog/da9052_wdt.c +++ b/drivers/watchdog/da9052_wdt.c @@ -176,14 +176,7 @@ static int da9052_wdt_probe(struct platform_device *pdev) return ret; } - ret = devm_watchdog_register_device(dev, &driver_data->wdt); - if (ret != 0) { - dev_err(da9052->dev, "watchdog_register_device() failed: %d\n", - ret); - return ret; - } - - return ret; + return devm_watchdog_register_device(dev, &driver_data->wdt); } static struct platform_driver da9052_wdt_driver = { diff --git a/drivers/watchdog/da9062_wdt.c b/drivers/watchdog/da9062_wdt.c index aac749cfaccb..e149e66a6ea9 100644 --- a/drivers/watchdog/da9062_wdt.c +++ b/drivers/watchdog/da9062_wdt.c @@ -214,11 +214,8 @@ static int da9062_wdt_probe(struct platform_device *pdev) watchdog_set_drvdata(&wdt->wdtdev, wdt); ret = devm_watchdog_register_device(dev, &wdt->wdtdev); - if (ret < 0) { - dev_err(wdt->hw->dev, - "watchdog registration failed (%d)\n", ret); + if (ret < 0) return ret; - } return da9062_wdt_ping(&wdt->wdtdev); } diff --git a/drivers/watchdog/davinci_wdt.c b/drivers/watchdog/davinci_wdt.c index 7b2ee35b5ffd..2b3f3cd382ef 100644 --- a/drivers/watchdog/davinci_wdt.c +++ b/drivers/watchdog/davinci_wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/char/watchdog/davinci_wdt.c * @@ -5,10 +6,7 @@ * * Copyright (C) 2006-2013 Texas Instruments. * - * 2007 (c) MontaVista Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. + * 2007 (c) MontaVista Software, Inc. */ #include <linux/module.h> @@ -247,13 +245,7 @@ static int davinci_wdt_probe(struct platform_device *pdev) if (IS_ERR(davinci_wdt->base)) return PTR_ERR(davinci_wdt->base); - ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "cannot register watchdog device\n"); - return ret; - } - - return 0; + return devm_watchdog_register_device(dev, wdd); } static const struct of_device_id davinci_wdt_of_match[] = { diff --git a/drivers/watchdog/digicolor_wdt.c b/drivers/watchdog/digicolor_wdt.c index 8af6e9a67d0d..073d37867f47 100644 --- a/drivers/watchdog/digicolor_wdt.c +++ b/drivers/watchdog/digicolor_wdt.c @@ -118,7 +118,6 @@ static int dc_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct dc_wdt *wdt; - int ret; wdt = devm_kzalloc(dev, sizeof(struct dc_wdt), GFP_KERNEL); if (!wdt) @@ -141,13 +140,7 @@ static int dc_wdt_probe(struct platform_device *pdev) watchdog_set_restart_priority(&dc_wdt_wdd, 128); watchdog_init_timeout(&dc_wdt_wdd, timeout, dev); watchdog_stop_on_reboot(&dc_wdt_wdd); - ret = devm_watchdog_register_device(dev, &dc_wdt_wdd); - if (ret) { - dev_err(dev, "Failed to register watchdog device"); - return ret; - } - - return 0; + return devm_watchdog_register_device(dev, &dc_wdt_wdd); } static const struct of_device_id dc_wdt_of_match[] = { diff --git a/drivers/watchdog/ebc-c384_wdt.c b/drivers/watchdog/ebc-c384_wdt.c index c176f59fea28..8ef4b0df3855 100644 --- a/drivers/watchdog/ebc-c384_wdt.c +++ b/drivers/watchdog/ebc-c384_wdt.c @@ -2,15 +2,6 @@ /* * Watchdog timer driver for the WinSystems EBC-C384 * Copyright (C) 2016 William Breathitt Gray - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. */ #include <linux/device.h> #include <linux/dmi.h> diff --git a/drivers/watchdog/eurotechwdt.c b/drivers/watchdog/eurotechwdt.c index 89129e6fa9b6..3a83a48abcae 100644 --- a/drivers/watchdog/eurotechwdt.c +++ b/drivers/watchdog/eurotechwdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Eurotech CPU-1220/1410/1420 on board WDT driver * @@ -11,11 +12,6 @@ * (c) Copyright 1996-1997 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/ftwdt010_wdt.c b/drivers/watchdog/ftwdt010_wdt.c index d9626ef9b9ae..21dcc7765688 100644 --- a/drivers/watchdog/ftwdt010_wdt.c +++ b/drivers/watchdog/ftwdt010_wdt.c @@ -165,10 +165,8 @@ static int ftwdt010_wdt_probe(struct platform_device *pdev) } ret = devm_watchdog_register_device(dev, &gwdt->wdd); - if (ret) { - dev_err(dev, "failed to register watchdog\n"); + if (ret) return ret; - } /* Set up platform driver data */ platform_set_drvdata(pdev, gwdt); diff --git a/drivers/watchdog/gpio_wdt.c b/drivers/watchdog/gpio_wdt.c index 777de10f2a78..0923201ce874 100644 --- a/drivers/watchdog/gpio_wdt.c +++ b/drivers/watchdog/gpio_wdt.c @@ -13,6 +13,12 @@ #include <linux/platform_device.h> #include <linux/watchdog.h> +static bool nowayout = WATCHDOG_NOWAYOUT; +module_param(nowayout, bool, 0); +MODULE_PARM_DESC(nowayout, + "Watchdog cannot be stopped once started (default=" + __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); + #define SOFT_TIMEOUT_MIN 1 #define SOFT_TIMEOUT_DEF 60 @@ -151,6 +157,7 @@ static int gpio_wdt_probe(struct platform_device *pdev) priv->wdd.timeout = SOFT_TIMEOUT_DEF; watchdog_init_timeout(&priv->wdd, 0, dev); + watchdog_set_nowayout(&priv->wdd, nowayout); watchdog_stop_on_reboot(&priv->wdd); diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c index 8a90f159ffb1..7d34bcf1c45b 100644 --- a/drivers/watchdog/hpwdt.c +++ b/drivers/watchdog/hpwdt.c @@ -22,10 +22,11 @@ #include <linux/watchdog.h> #include <asm/nmi.h> -#define HPWDT_VERSION "2.0.2" +#define HPWDT_VERSION "2.0.3" #define SECS_TO_TICKS(secs) ((secs) * 1000 / 128) #define TICKS_TO_SECS(ticks) ((ticks) * 128 / 1000) -#define HPWDT_MAX_TIMER TICKS_TO_SECS(65535) +#define HPWDT_MAX_TICKS 65535 +#define HPWDT_MAX_TIMER TICKS_TO_SECS(HPWDT_MAX_TICKS) #define DEFAULT_MARGIN 30 #define PRETIMEOUT_SEC 9 @@ -33,6 +34,7 @@ static bool ilo5; static unsigned int soft_margin = DEFAULT_MARGIN; /* in seconds */ static bool nowayout = WATCHDOG_NOWAYOUT; static bool pretimeout = IS_ENABLED(CONFIG_HPWDT_NMI_DECODING); +static int kdumptimeout = -1; static void __iomem *pci_mem_addr; /* the PCI-memory address */ static unsigned long __iomem *hpwdt_nmistat; @@ -52,15 +54,21 @@ static const struct pci_device_id hpwdt_blacklist[] = { {0}, /* terminate list */ }; +static struct watchdog_device hpwdt_dev; /* * Watchdog operations */ +static int hpwdt_hw_is_running(void) +{ + return ioread8(hpwdt_timer_con) & 0x01; +} + static int hpwdt_start(struct watchdog_device *wdd) { int control = 0x81 | (pretimeout ? 0x4 : 0); - int reload = SECS_TO_TICKS(wdd->timeout); + int reload = SECS_TO_TICKS(min(wdd->timeout, wdd->max_hw_heartbeat_ms/1000)); - dev_dbg(wdd->parent, "start watchdog 0x%08x:0x%02x\n", reload, control); + dev_dbg(wdd->parent, "start watchdog 0x%08x:0x%08x:0x%02x\n", wdd->timeout, reload, control); iowrite16(reload, hpwdt_timer_reg); iowrite8(control, hpwdt_timer_con); @@ -85,12 +93,18 @@ static int hpwdt_stop_core(struct watchdog_device *wdd) return 0; } +static void hpwdt_ping_ticks(int val) +{ + val = min(val, HPWDT_MAX_TICKS); + iowrite16(val, hpwdt_timer_reg); +} + static int hpwdt_ping(struct watchdog_device *wdd) { - int reload = SECS_TO_TICKS(wdd->timeout); + int reload = SECS_TO_TICKS(min(wdd->timeout, wdd->max_hw_heartbeat_ms/1000)); - dev_dbg(wdd->parent, "ping watchdog 0x%08x\n", reload); - iowrite16(reload, hpwdt_timer_reg); + dev_dbg(wdd->parent, "ping watchdog 0x%08x:0x%08x\n", wdd->timeout, reload); + hpwdt_ping_ticks(reload); return 0; } @@ -166,7 +180,14 @@ static int hpwdt_pretimeout(unsigned int ulReason, struct pt_regs *regs) if (ilo5 && !pretimeout && !mynmi) return NMI_DONE; - hpwdt_stop(); + if (kdumptimeout < 0) + hpwdt_stop(); + else if (kdumptimeout == 0) + ; + else { + unsigned int val = max((unsigned int)kdumptimeout, hpwdt_dev.timeout); + hpwdt_ping_ticks(SECS_TO_TICKS(val)); + } hex_byte_pack(panic_msg, mynmi); nmi_panic(regs, panic_msg); @@ -204,9 +225,9 @@ static struct watchdog_device hpwdt_dev = { .info = &ident, .ops = &hpwdt_ops, .min_timeout = 1, - .max_timeout = HPWDT_MAX_TIMER, .timeout = DEFAULT_MARGIN, .pretimeout = PRETIMEOUT_SEC, + .max_hw_heartbeat_ms = HPWDT_MAX_TIMER * 1000, }; @@ -298,14 +319,18 @@ static int hpwdt_init_one(struct pci_dev *dev, hpwdt_timer_reg = pci_mem_addr + 0x70; hpwdt_timer_con = pci_mem_addr + 0x72; - /* Make sure that timer is disabled until /dev/watchdog is opened */ - hpwdt_stop(); + /* Have the core update running timer until user space is ready */ + if (hpwdt_hw_is_running()) { + dev_info(&dev->dev, "timer is running\n"); + set_bit(WDOG_HW_RUNNING, &hpwdt_dev.status); + } /* Initialize NMI Decoding functionality */ retval = hpwdt_init_nmi_decoding(dev); if (retval != 0) goto error_init_nmi_decoding; + watchdog_stop_on_unregister(&hpwdt_dev); watchdog_set_nowayout(&hpwdt_dev, nowayout); watchdog_init_timeout(&hpwdt_dev, soft_margin, NULL); @@ -314,13 +339,12 @@ static int hpwdt_init_one(struct pci_dev *dev, pretimeout = 0; } hpwdt_dev.pretimeout = pretimeout ? PRETIMEOUT_SEC : 0; + kdumptimeout = min(kdumptimeout, HPWDT_MAX_TIMER); hpwdt_dev.parent = &dev->dev; retval = watchdog_register_device(&hpwdt_dev); - if (retval < 0) { - dev_err(&dev->dev, "watchdog register failed: %d.\n", retval); + if (retval < 0) goto error_wd_register; - } dev_info(&dev->dev, "HPE Watchdog Timer Driver: Version: %s\n", HPWDT_VERSION); @@ -328,6 +352,7 @@ static int hpwdt_init_one(struct pci_dev *dev, hpwdt_dev.timeout, nowayout); dev_info(&dev->dev, "pretimeout: %s.\n", pretimeout ? "on" : "off"); + dev_info(&dev->dev, "kdumptimeout: %d.\n", kdumptimeout); if (dev->subsystem_vendor == PCI_VENDOR_ID_HP_3PAR) ilo5 = true; @@ -345,9 +370,6 @@ error_pci_iomap: static void hpwdt_exit(struct pci_dev *dev) { - if (!nowayout) - hpwdt_stop(); - watchdog_unregister_device(&hpwdt_dev); hpwdt_exit_nmi_decoding(); pci_iounmap(dev, pci_mem_addr); @@ -376,6 +398,9 @@ module_param(nowayout, bool, 0); MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); +module_param(kdumptimeout, int, 0444); +MODULE_PARM_DESC(kdumptimeout, "Timeout applied for crash kernel transition in seconds"); + #ifdef CONFIG_HPWDT_NMI_DECODING module_param(pretimeout, bool, 0); MODULE_PARM_DESC(pretimeout, "Watchdog pretimeout enabled"); diff --git a/drivers/watchdog/i6300esb.c b/drivers/watchdog/i6300esb.c index f98f35a05896..a30835f547b3 100644 --- a/drivers/watchdog/i6300esb.c +++ b/drivers/watchdog/i6300esb.c @@ -315,11 +315,8 @@ static int esb_probe(struct pci_dev *pdev, /* Register the watchdog so that userspace has access to it */ ret = watchdog_register_device(&edev->wdd); - if (ret != 0) { - dev_err(&pdev->dev, - "cannot register watchdog device (err=%d)\n", ret); + if (ret != 0) goto err_unmap; - } dev_info(&pdev->dev, "initialized. heartbeat=%d sec (nowayout=%d)\n", edev->wdd.timeout, nowayout); diff --git a/drivers/watchdog/iTCO_vendor_support.c b/drivers/watchdog/iTCO_vendor_support.c index 68a9d9cc2eb8..4f1b96f59349 100644 --- a/drivers/watchdog/iTCO_vendor_support.c +++ b/drivers/watchdog/iTCO_vendor_support.c @@ -1,13 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * intel TCO vendor specific watchdog driver support * * (c) Copyright 2006-2009 Wim Van Sebroeck <wim@iguana.be>. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Wim Van Sebroeck nor Iguana vzw. admit liability nor * provide warranty for any of this software. This material is * provided "AS-IS" and at no charge. @@ -216,4 +212,3 @@ MODULE_AUTHOR("Wim Van Sebroeck <wim@iguana.be>, " MODULE_DESCRIPTION("Intel TCO Vendor Specific WatchDog Timer Driver Support"); MODULE_VERSION(DRV_VERSION); MODULE_LICENSE("GPL"); - diff --git a/drivers/watchdog/iTCO_wdt.c b/drivers/watchdog/iTCO_wdt.c index 89cea6ce9a08..c559f706ae7e 100644 --- a/drivers/watchdog/iTCO_wdt.c +++ b/drivers/watchdog/iTCO_wdt.c @@ -1,13 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * intel TCO Watchdog Driver * * (c) Copyright 2006-2011 Wim Van Sebroeck <wim@iguana.be>. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Wim Van Sebroeck nor Iguana vzw. admit liability nor * provide warranty for any of this software. This material is * provided "AS-IS" and at no charge. diff --git a/drivers/watchdog/ib700wdt.c b/drivers/watchdog/ib700wdt.c index 30d6cec582af..92fd7f33bc4d 100644 --- a/drivers/watchdog/ib700wdt.c +++ b/drivers/watchdog/ib700wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * IB700 Single Board Computer WDT driver * @@ -14,11 +15,6 @@ * (c) Copyright 1996 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/ie6xx_wdt.c b/drivers/watchdog/ie6xx_wdt.c index 508fbefce9f6..8f28993fab8b 100644 --- a/drivers/watchdog/ie6xx_wdt.c +++ b/drivers/watchdog/ie6xx_wdt.c @@ -66,7 +66,7 @@ MODULE_PARM_DESC(resetmode, static struct { unsigned short sch_wdtba; - struct spinlock unlock_sequence; + spinlock_t unlock_sequence; #ifdef CONFIG_DEBUG_FS struct dentry *debugfs; #endif @@ -254,12 +254,8 @@ static int ie6xx_wdt_probe(struct platform_device *pdev) ie6xx_wdt_debugfs_init(); ret = watchdog_register_device(&ie6xx_wdt_dev); - if (ret) { - dev_err(&pdev->dev, - "Watchdog timer: cannot register device (err =%d)\n", - ret); + if (ret) goto misc_register_error; - } return 0; diff --git a/drivers/watchdog/imx2_wdt.c b/drivers/watchdog/imx2_wdt.c index a606005dd65f..32af3974e6bb 100644 --- a/drivers/watchdog/imx2_wdt.c +++ b/drivers/watchdog/imx2_wdt.c @@ -316,10 +316,8 @@ static int __init imx2_wdt_probe(struct platform_device *pdev) regmap_write(wdev->regmap, IMX2_WDT_WMCR, 0); ret = watchdog_register_device(wdog); - if (ret) { - dev_err(&pdev->dev, "cannot register watchdog device\n"); + if (ret) goto disable_clk; - } dev_info(&pdev->dev, "timeout %d sec (nowayout=%d)\n", wdog->timeout, nowayout); diff --git a/drivers/watchdog/imx_sc_wdt.c b/drivers/watchdog/imx_sc_wdt.c index 49848b66186c..78eaaf75a263 100644 --- a/drivers/watchdog/imx_sc_wdt.c +++ b/drivers/watchdog/imx_sc_wdt.c @@ -4,6 +4,7 @@ */ #include <linux/arm-smccc.h> +#include <linux/firmware/imx/sci.h> #include <linux/io.h> #include <linux/init.h> #include <linux/kernel.h> @@ -33,11 +34,19 @@ #define SC_TIMER_WDOG_ACTION_PARTITION 0 +#define SC_IRQ_WDOG 1 +#define SC_IRQ_GROUP_WDOG 1 + static bool nowayout = WATCHDOG_NOWAYOUT; module_param(nowayout, bool, 0000); MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); +struct imx_sc_wdt_device { + struct watchdog_device wdd; + struct notifier_block wdt_notifier; +}; + static int imx_sc_wdt_ping(struct watchdog_device *wdog) { struct arm_smccc_res res; @@ -85,24 +94,66 @@ static int imx_sc_wdt_set_timeout(struct watchdog_device *wdog, return res.a0 ? -EACCES : 0; } +static int imx_sc_wdt_set_pretimeout(struct watchdog_device *wdog, + unsigned int pretimeout) +{ + struct arm_smccc_res res; + + arm_smccc_smc(IMX_SIP_TIMER, IMX_SIP_TIMER_SET_PRETIME_WDOG, + pretimeout * 1000, 0, 0, 0, 0, 0, &res); + if (res.a0) + return -EACCES; + + wdog->pretimeout = pretimeout; + + return 0; +} + +static int imx_sc_wdt_notify(struct notifier_block *nb, + unsigned long event, void *group) +{ + struct imx_sc_wdt_device *imx_sc_wdd = + container_of(nb, + struct imx_sc_wdt_device, + wdt_notifier); + + if (event & SC_IRQ_WDOG && + *(u8 *)group == SC_IRQ_GROUP_WDOG) + watchdog_notify_pretimeout(&imx_sc_wdd->wdd); + + return 0; +} + +static void imx_sc_wdt_action(void *data) +{ + struct notifier_block *wdt_notifier = data; + + imx_scu_irq_unregister_notifier(wdt_notifier); + imx_scu_irq_group_enable(SC_IRQ_GROUP_WDOG, + SC_IRQ_WDOG, + false); +} + static const struct watchdog_ops imx_sc_wdt_ops = { .owner = THIS_MODULE, .start = imx_sc_wdt_start, .stop = imx_sc_wdt_stop, .ping = imx_sc_wdt_ping, .set_timeout = imx_sc_wdt_set_timeout, + .set_pretimeout = imx_sc_wdt_set_pretimeout, }; -static const struct watchdog_info imx_sc_wdt_info = { +static struct watchdog_info imx_sc_wdt_info = { .identity = "i.MX SC watchdog timer", .options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | - WDIOF_MAGICCLOSE | WDIOF_PRETIMEOUT, + WDIOF_MAGICCLOSE, }; static int imx_sc_wdt_probe(struct platform_device *pdev) { + struct imx_sc_wdt_device *imx_sc_wdd; + struct watchdog_device *wdog; struct device *dev = &pdev->dev; - struct watchdog_device *imx_sc_wdd; int ret; imx_sc_wdd = devm_kzalloc(dev, sizeof(*imx_sc_wdd), GFP_KERNEL); @@ -111,42 +162,70 @@ static int imx_sc_wdt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, imx_sc_wdd); - imx_sc_wdd->info = &imx_sc_wdt_info; - imx_sc_wdd->ops = &imx_sc_wdt_ops; - imx_sc_wdd->min_timeout = 1; - imx_sc_wdd->max_timeout = MAX_TIMEOUT; - imx_sc_wdd->parent = dev; - imx_sc_wdd->timeout = DEFAULT_TIMEOUT; - - watchdog_init_timeout(imx_sc_wdd, 0, dev); - watchdog_stop_on_reboot(imx_sc_wdd); - watchdog_stop_on_unregister(imx_sc_wdd); + wdog = &imx_sc_wdd->wdd; + wdog->info = &imx_sc_wdt_info; + wdog->ops = &imx_sc_wdt_ops; + wdog->min_timeout = 1; + wdog->max_timeout = MAX_TIMEOUT; + wdog->parent = dev; + wdog->timeout = DEFAULT_TIMEOUT; + + watchdog_init_timeout(wdog, 0, dev); + watchdog_stop_on_reboot(wdog); + watchdog_stop_on_unregister(wdog); + + ret = devm_watchdog_register_device(dev, wdog); + + if (ret) { + dev_err(dev, "Failed to register watchdog device\n"); + return ret; + } + + ret = imx_scu_irq_group_enable(SC_IRQ_GROUP_WDOG, + SC_IRQ_WDOG, + true); + if (ret) { + dev_warn(dev, "Enable irq failed, pretimeout NOT supported\n"); + return 0; + } - ret = devm_watchdog_register_device(dev, imx_sc_wdd); + imx_sc_wdd->wdt_notifier.notifier_call = imx_sc_wdt_notify; + ret = imx_scu_irq_register_notifier(&imx_sc_wdd->wdt_notifier); if (ret) { - dev_err(dev, "Failed to register watchdog device\n"); - return ret; + imx_scu_irq_group_enable(SC_IRQ_GROUP_WDOG, + SC_IRQ_WDOG, + false); + dev_warn(dev, + "Register irq notifier failed, pretimeout NOT supported\n"); + return 0; } + ret = devm_add_action_or_reset(dev, imx_sc_wdt_action, + &imx_sc_wdd->wdt_notifier); + if (!ret) + imx_sc_wdt_info.options |= WDIOF_PRETIMEOUT; + else + dev_warn(dev, "Add action failed, pretimeout NOT supported\n"); + return 0; } static int __maybe_unused imx_sc_wdt_suspend(struct device *dev) { - struct watchdog_device *imx_sc_wdd = dev_get_drvdata(dev); + struct imx_sc_wdt_device *imx_sc_wdd = dev_get_drvdata(dev); - if (watchdog_active(imx_sc_wdd)) - imx_sc_wdt_stop(imx_sc_wdd); + if (watchdog_active(&imx_sc_wdd->wdd)) + imx_sc_wdt_stop(&imx_sc_wdd->wdd); return 0; } static int __maybe_unused imx_sc_wdt_resume(struct device *dev) { - struct watchdog_device *imx_sc_wdd = dev_get_drvdata(dev); + struct imx_sc_wdt_device *imx_sc_wdd = dev_get_drvdata(dev); - if (watchdog_active(imx_sc_wdd)) - imx_sc_wdt_start(imx_sc_wdd); + if (watchdog_active(&imx_sc_wdd->wdd)) + imx_sc_wdt_start(&imx_sc_wdd->wdd); return 0; } diff --git a/drivers/watchdog/intel-mid_wdt.c b/drivers/watchdog/intel-mid_wdt.c index b2463f8276e6..2cdbd37c700c 100644 --- a/drivers/watchdog/intel-mid_wdt.c +++ b/drivers/watchdog/intel-mid_wdt.c @@ -161,10 +161,8 @@ static int mid_wdt_probe(struct platform_device *pdev) set_bit(WDOG_HW_RUNNING, &wdt_dev->status); ret = devm_watchdog_register_device(dev, wdt_dev); - if (ret) { - dev_err(dev, "error registering watchdog device\n"); + if (ret) return ret; - } dev_info(dev, "Intel MID watchdog device probed\n"); diff --git a/drivers/watchdog/jz4740_wdt.c b/drivers/watchdog/jz4740_wdt.c index 313358b2e0b1..d4a90916dd38 100644 --- a/drivers/watchdog/jz4740_wdt.c +++ b/drivers/watchdog/jz4740_wdt.c @@ -4,6 +4,7 @@ * JZ4740 Watchdog driver */ +#include <linux/mfd/ingenic-tcu.h> #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/types.h> @@ -19,23 +20,16 @@ #include <asm/mach-jz4740/timer.h> -#define JZ_REG_WDT_TIMER_DATA 0x0 -#define JZ_REG_WDT_COUNTER_ENABLE 0x4 -#define JZ_REG_WDT_TIMER_COUNTER 0x8 -#define JZ_REG_WDT_TIMER_CONTROL 0xC - #define JZ_WDT_CLOCK_PCLK 0x1 #define JZ_WDT_CLOCK_RTC 0x2 #define JZ_WDT_CLOCK_EXT 0x4 -#define JZ_WDT_CLOCK_DIV_SHIFT 3 - -#define JZ_WDT_CLOCK_DIV_1 (0 << JZ_WDT_CLOCK_DIV_SHIFT) -#define JZ_WDT_CLOCK_DIV_4 (1 << JZ_WDT_CLOCK_DIV_SHIFT) -#define JZ_WDT_CLOCK_DIV_16 (2 << JZ_WDT_CLOCK_DIV_SHIFT) -#define JZ_WDT_CLOCK_DIV_64 (3 << JZ_WDT_CLOCK_DIV_SHIFT) -#define JZ_WDT_CLOCK_DIV_256 (4 << JZ_WDT_CLOCK_DIV_SHIFT) -#define JZ_WDT_CLOCK_DIV_1024 (5 << JZ_WDT_CLOCK_DIV_SHIFT) +#define JZ_WDT_CLOCK_DIV_1 (0 << TCU_TCSR_PRESCALE_LSB) +#define JZ_WDT_CLOCK_DIV_4 (1 << TCU_TCSR_PRESCALE_LSB) +#define JZ_WDT_CLOCK_DIV_16 (2 << TCU_TCSR_PRESCALE_LSB) +#define JZ_WDT_CLOCK_DIV_64 (3 << TCU_TCSR_PRESCALE_LSB) +#define JZ_WDT_CLOCK_DIV_256 (4 << TCU_TCSR_PRESCALE_LSB) +#define JZ_WDT_CLOCK_DIV_1024 (5 << TCU_TCSR_PRESCALE_LSB) #define DEFAULT_HEARTBEAT 5 #define MAX_HEARTBEAT 2048 @@ -63,7 +57,7 @@ static int jz4740_wdt_ping(struct watchdog_device *wdt_dev) { struct jz4740_wdt_drvdata *drvdata = watchdog_get_drvdata(wdt_dev); - writew(0x0, drvdata->base + JZ_REG_WDT_TIMER_COUNTER); + writew(0x0, drvdata->base + TCU_REG_WDT_TCNT); return 0; } @@ -74,6 +68,7 @@ static int jz4740_wdt_set_timeout(struct watchdog_device *wdt_dev, unsigned int rtc_clk_rate; unsigned int timeout_value; unsigned short clock_div = JZ_WDT_CLOCK_DIV_1; + u8 tcer; rtc_clk_rate = clk_get_rate(drvdata->rtc_clk); @@ -86,18 +81,19 @@ static int jz4740_wdt_set_timeout(struct watchdog_device *wdt_dev, break; } timeout_value >>= 2; - clock_div += (1 << JZ_WDT_CLOCK_DIV_SHIFT); + clock_div += (1 << TCU_TCSR_PRESCALE_LSB); } - writeb(0x0, drvdata->base + JZ_REG_WDT_COUNTER_ENABLE); - writew(clock_div, drvdata->base + JZ_REG_WDT_TIMER_CONTROL); + tcer = readb(drvdata->base + TCU_REG_WDT_TCER); + writeb(0x0, drvdata->base + TCU_REG_WDT_TCER); + writew(clock_div, drvdata->base + TCU_REG_WDT_TCSR); - writew((u16)timeout_value, drvdata->base + JZ_REG_WDT_TIMER_DATA); - writew(0x0, drvdata->base + JZ_REG_WDT_TIMER_COUNTER); - writew(clock_div | JZ_WDT_CLOCK_RTC, - drvdata->base + JZ_REG_WDT_TIMER_CONTROL); + writew((u16)timeout_value, drvdata->base + TCU_REG_WDT_TDR); + writew(0x0, drvdata->base + TCU_REG_WDT_TCNT); + writew(clock_div | JZ_WDT_CLOCK_RTC, drvdata->base + TCU_REG_WDT_TCSR); - writeb(0x1, drvdata->base + JZ_REG_WDT_COUNTER_ENABLE); + if (tcer & TCU_WDT_TCER_TCEN) + writeb(TCU_WDT_TCER_TCEN, drvdata->base + TCU_REG_WDT_TCER); wdt_dev->timeout = new_timeout; return 0; @@ -105,9 +101,18 @@ static int jz4740_wdt_set_timeout(struct watchdog_device *wdt_dev, static int jz4740_wdt_start(struct watchdog_device *wdt_dev) { + struct jz4740_wdt_drvdata *drvdata = watchdog_get_drvdata(wdt_dev); + u8 tcer; + + tcer = readb(drvdata->base + TCU_REG_WDT_TCER); + jz4740_timer_enable_watchdog(); jz4740_wdt_set_timeout(wdt_dev, wdt_dev->timeout); + /* Start watchdog if it wasn't started already */ + if (!(tcer & TCU_WDT_TCER_TCEN)) + writeb(TCU_WDT_TCER_TCEN, drvdata->base + TCU_REG_WDT_TCER); + return 0; } @@ -115,7 +120,7 @@ static int jz4740_wdt_stop(struct watchdog_device *wdt_dev) { struct jz4740_wdt_drvdata *drvdata = watchdog_get_drvdata(wdt_dev); - writeb(0x0, drvdata->base + JZ_REG_WDT_COUNTER_ENABLE); + writeb(0x0, drvdata->base + TCU_REG_WDT_TCER); jz4740_timer_disable_watchdog(); return 0; @@ -187,11 +192,7 @@ static int jz4740_wdt_probe(struct platform_device *pdev) return PTR_ERR(drvdata->rtc_clk); } - ret = devm_watchdog_register_device(dev, &drvdata->wdt); - if (ret < 0) - return ret; - - return 0; + return devm_watchdog_register_device(dev, &drvdata->wdt); } static struct platform_driver jz4740_wdt_driver = { diff --git a/drivers/watchdog/loongson1_wdt.c b/drivers/watchdog/loongson1_wdt.c index c8c2b8a88fc2..bb3d075c0633 100644 --- a/drivers/watchdog/loongson1_wdt.c +++ b/drivers/watchdog/loongson1_wdt.c @@ -132,10 +132,8 @@ static int ls1x_wdt_probe(struct platform_device *pdev) watchdog_set_drvdata(ls1x_wdt, drvdata); err = devm_watchdog_register_device(dev, &drvdata->wdt); - if (err) { - dev_err(dev, "failed to register watchdog device\n"); + if (err) return err; - } platform_set_drvdata(pdev, drvdata); diff --git a/drivers/watchdog/max77620_wdt.c b/drivers/watchdog/max77620_wdt.c index 9937f9fccd2e..be6a53c30002 100644 --- a/drivers/watchdog/max77620_wdt.c +++ b/drivers/watchdog/max77620_wdt.c @@ -182,13 +182,7 @@ static int max77620_wdt_probe(struct platform_device *pdev) watchdog_set_drvdata(wdt_dev, wdt); watchdog_stop_on_unregister(wdt_dev); - ret = devm_watchdog_register_device(dev, wdt_dev); - if (ret < 0) { - dev_err(dev, "watchdog registration failed: %d\n", ret); - return ret; - } - - return 0; + return devm_watchdog_register_device(dev, wdt_dev); } static const struct platform_device_id max77620_wdt_devtype[] = { diff --git a/drivers/watchdog/mei_wdt.c b/drivers/watchdog/mei_wdt.c index 96a770938ff0..5391bf3e6b11 100644 --- a/drivers/watchdog/mei_wdt.c +++ b/drivers/watchdog/mei_wdt.c @@ -384,10 +384,8 @@ static int mei_wdt_register(struct mei_wdt *wdt) watchdog_stop_on_reboot(&wdt->wdd); ret = watchdog_register_device(&wdt->wdd); - if (ret) { - dev_err(dev, "unable to register watchdog device = %d.\n", ret); + if (ret) watchdog_set_drvdata(&wdt->wdd, NULL); - } wdt->state = MEI_WDT_IDLE; diff --git a/drivers/watchdog/mena21_wdt.c b/drivers/watchdog/mena21_wdt.c index e9ca4e0e25dc..99d2359d5a8a 100644 --- a/drivers/watchdog/mena21_wdt.c +++ b/drivers/watchdog/mena21_wdt.c @@ -190,10 +190,8 @@ static int a21_wdt_probe(struct platform_device *pdev) dev_set_drvdata(dev, drv); ret = devm_watchdog_register_device(dev, &a21_wdt); - if (ret) { - dev_err(dev, "Cannot register watchdog device\n"); + if (ret) return ret; - } dev_info(dev, "MEN A21 watchdog timer driver enabled\n"); diff --git a/drivers/watchdog/menf21bmc_wdt.c b/drivers/watchdog/menf21bmc_wdt.c index 7766d7361d3b..81ebdfc371f4 100644 --- a/drivers/watchdog/menf21bmc_wdt.c +++ b/drivers/watchdog/menf21bmc_wdt.c @@ -152,10 +152,8 @@ static int menf21bmc_wdt_probe(struct platform_device *pdev) } ret = devm_watchdog_register_device(dev, &drv_data->wdt); - if (ret) { - dev_err(dev, "failed to register Watchdog device\n"); + if (ret) return ret; - } dev_info(dev, "MEN 14F021P00 BMC Watchdog device enabled\n"); diff --git a/drivers/watchdog/mpc8xxx_wdt.c b/drivers/watchdog/mpc8xxx_wdt.c index b6ffad421bd0..3fc457bc16db 100644 --- a/drivers/watchdog/mpc8xxx_wdt.c +++ b/drivers/watchdog/mpc8xxx_wdt.c @@ -201,11 +201,8 @@ static int mpc8xxx_wdt_probe(struct platform_device *ofdev) ddata->wdd.timeout = ddata->wdd.min_timeout; ret = devm_watchdog_register_device(dev, &ddata->wdd); - if (ret) { - dev_err(dev, "cannot register watchdog device (err=%d)\n", - ret); + if (ret) return ret; - } dev_info(dev, "WDT driver for MPC8xxx initialized. mode:%s timeout=%d sec\n", diff --git a/drivers/watchdog/mv64x60_wdt.c b/drivers/watchdog/mv64x60_wdt.c index c785f4f0a196..74bf7144a970 100644 --- a/drivers/watchdog/mv64x60_wdt.c +++ b/drivers/watchdog/mv64x60_wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * mv64x60_wdt.c - MV64X60 (Marvell Discovery) watchdog userspace interface * @@ -9,10 +10,7 @@ * * Derived from mpc8xx_wdt.c, with the following copyright. * - * 2002 (c) Florian Schirmer <jolt@tuxbox.org> This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. + * 2002 (c) Florian Schirmer <jolt@tuxbox.org> */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/watchdog/ni903x_wdt.c b/drivers/watchdog/ni903x_wdt.c index 60f5608af2a8..4cebad324b20 100644 --- a/drivers/watchdog/ni903x_wdt.c +++ b/drivers/watchdog/ni903x_wdt.c @@ -211,10 +211,8 @@ static int ni903x_acpi_add(struct acpi_device *device) watchdog_init_timeout(wdd, timeout, dev); ret = watchdog_register_device(wdd); - if (ret) { - dev_err(dev, "failed to register watchdog\n"); + if (ret) return ret; - } /* Switch from boot mode to user mode */ outb(NIWD_CONTROL_RESET | NIWD_CONTROL_MODE, diff --git a/drivers/watchdog/nic7018_wdt.c b/drivers/watchdog/nic7018_wdt.c index 2e1a2a3d4ec9..2a46cc662943 100644 --- a/drivers/watchdog/nic7018_wdt.c +++ b/drivers/watchdog/nic7018_wdt.c @@ -210,7 +210,6 @@ static int nic7018_probe(struct platform_device *pdev) ret = watchdog_register_device(wdd); if (ret) { outb(LOCK, wdt->io_base + WDT_REG_LOCK); - dev_err(dev, "failed to register watchdog\n"); return ret; } diff --git a/drivers/watchdog/npcm_wdt.c b/drivers/watchdog/npcm_wdt.c index 9d6c1689b12c..9c773c3d6d5d 100644 --- a/drivers/watchdog/npcm_wdt.c +++ b/drivers/watchdog/npcm_wdt.c @@ -220,10 +220,8 @@ static int npcm_wdt_probe(struct platform_device *pdev) return ret; ret = devm_watchdog_register_device(dev, &wdt->wdd); - if (ret) { - dev_err(dev, "failed to register watchdog\n"); + if (ret) return ret; - } dev_info(dev, "NPCM watchdog driver enabled\n"); diff --git a/drivers/watchdog/nv_tco.h b/drivers/watchdog/nv_tco.h index c2d1d04e055b..d325e528010f 100644 --- a/drivers/watchdog/nv_tco.h +++ b/drivers/watchdog/nv_tco.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ /* * nv_tco: TCO timer driver for nVidia chipsets. * @@ -10,11 +11,6 @@ * Reserved. * http://www.kernelconcepts.de * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither kernel concepts nor Nils Faerber admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c index 0ec419a3f7ed..fde9e739b436 100644 --- a/drivers/watchdog/octeon-wdt-main.c +++ b/drivers/watchdog/octeon-wdt-main.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Octeon Watchdog driver * @@ -10,22 +11,12 @@ * (c) Copyright 1996-1997 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. * * (c) Copyright 1995 Alan Cox <alan@lxorguk.ukuu.org.uk> * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * * The OCTEON watchdog has a maximum timeout of 2^32 * io_clock. * For most systems this is less than 10 seconds, so to allow for * software to request longer watchdog heartbeats, we maintain software diff --git a/drivers/watchdog/of_xilinx_wdt.c b/drivers/watchdog/of_xilinx_wdt.c index 03786992b701..7fe4f7c3f7ce 100644 --- a/drivers/watchdog/of_xilinx_wdt.c +++ b/drivers/watchdog/of_xilinx_wdt.c @@ -238,10 +238,8 @@ static int xwdt_probe(struct platform_device *pdev) } rc = devm_watchdog_register_device(dev, xilinx_wdt_wdd); - if (rc) { - dev_err(dev, "Cannot register watchdog (err=%d)\n", rc); + if (rc) return rc; - } clk_disable(xdev->clk); diff --git a/drivers/watchdog/omap_wdt.c b/drivers/watchdog/omap_wdt.c index d49688d93f6a..9b91882fe3c4 100644 --- a/drivers/watchdog/omap_wdt.c +++ b/drivers/watchdog/omap_wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * omap_wdt.c * @@ -6,10 +7,7 @@ * Author: MontaVista Software, Inc. * <gdavis@mvista.com> or <source@mvista.com> * - * 2003 (c) MontaVista Software, Inc. This file is licensed under the - * terms of the GNU General Public License version 2. This program is - * licensed "as is" without any warranty of any kind, whether express - * or implied. + * 2003 (c) MontaVista Software, Inc. * * History: * diff --git a/drivers/watchdog/omap_wdt.h b/drivers/watchdog/omap_wdt.h index 42f31ec5e90d..950b4643f3e7 100644 --- a/drivers/watchdog/omap_wdt.h +++ b/drivers/watchdog/omap_wdt.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ /* * linux/drivers/char/watchdog/omap_wdt.h * @@ -5,26 +6,6 @@ * OMAP Watchdog timer register definitions * * Copyright (C) 2004 Texas Instruments. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef _OMAP_WATCHDOG_H diff --git a/drivers/watchdog/pc87413_wdt.c b/drivers/watchdog/pc87413_wdt.c index ca21d6c240a3..2af1a8b3f973 100644 --- a/drivers/watchdog/pc87413_wdt.c +++ b/drivers/watchdog/pc87413_wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * NS pc87413-wdt Watchdog Timer driver for Linux 2.6.x.x * @@ -6,11 +7,6 @@ * (C) Copyright 2006 Sven Anders, <anders@anduras.de> * and Marcus Junker, <junker@anduras.de> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Sven Anders, Marcus Junker nor ANDURAS AG * admit liability nor provide warranty for any of this software. * This material is provided "AS-IS" and at no charge. diff --git a/drivers/watchdog/pcwd_pci.c b/drivers/watchdog/pcwd_pci.c index 5773d2591d3f..e30c1f762045 100644 --- a/drivers/watchdog/pcwd_pci.c +++ b/drivers/watchdog/pcwd_pci.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Berkshire PCI-PC Watchdog Card Driver * @@ -10,11 +11,6 @@ * Matt Domsch <Matt_Domsch@dell.com>, * Rob Radez <rob@osinvestor.com> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Wim Van Sebroeck nor Iguana vzw. admit liability nor * provide warranty for any of this software. This material is * provided "AS-IS" and at no charge. diff --git a/drivers/watchdog/pcwd_usb.c b/drivers/watchdog/pcwd_usb.c index 5de6182dae33..6727f8ab2d18 100644 --- a/drivers/watchdog/pcwd_usb.c +++ b/drivers/watchdog/pcwd_usb.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Berkshire USB-PC Watchdog Card Driver * @@ -10,11 +11,6 @@ * Rob Radez <rob@osinvestor.com>, * Greg Kroah-Hartman <greg@kroah.com> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Wim Van Sebroeck nor Iguana vzw. admit liability nor * provide warranty for any of this software. This material is * provided "AS-IS" and at no charge. diff --git a/drivers/watchdog/pic32-dmt.c b/drivers/watchdog/pic32-dmt.c index 4f2aca78f13a..f43062b3c4c8 100644 --- a/drivers/watchdog/pic32-dmt.c +++ b/drivers/watchdog/pic32-dmt.c @@ -212,10 +212,8 @@ static int pic32_dmt_probe(struct platform_device *pdev) watchdog_set_drvdata(wdd, dmt); ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "watchdog register failed, err %d\n", ret); + if (ret) return ret; - } platform_set_drvdata(pdev, wdd); return 0; diff --git a/drivers/watchdog/pic32-wdt.c b/drivers/watchdog/pic32-wdt.c index 5ecdd880f0b7..41715d68d9e9 100644 --- a/drivers/watchdog/pic32-wdt.c +++ b/drivers/watchdog/pic32-wdt.c @@ -221,10 +221,8 @@ static int pic32_wdt_drv_probe(struct platform_device *pdev) watchdog_set_drvdata(wdd, wdt); ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "watchdog register failed, err %d\n", ret); + if (ret) return ret; - } platform_set_drvdata(pdev, wdd); diff --git a/drivers/watchdog/pnx4008_wdt.c b/drivers/watchdog/pnx4008_wdt.c index d9e03544aeae..7b446b696f2b 100644 --- a/drivers/watchdog/pnx4008_wdt.c +++ b/drivers/watchdog/pnx4008_wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * drivers/char/watchdog/pnx4008_wdt.c * @@ -11,10 +12,6 @@ * 2005-2006 (c) MontaVista Software, Inc. * * (C) 2012 Wolfram Sang, Pengutronix - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -221,10 +218,8 @@ static int pnx4008_wdt_probe(struct platform_device *pdev) set_bit(WDOG_HW_RUNNING, &pnx4008_wdd.status); ret = devm_watchdog_register_device(dev, &pnx4008_wdd); - if (ret < 0) { - dev_err(dev, "cannot register watchdog device\n"); + if (ret < 0) return ret; - } dev_info(dev, "heartbeat %d sec\n", pnx4008_wdd.timeout); diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c index fc0f7e5de38d..7be7f87be28f 100644 --- a/drivers/watchdog/qcom-wdt.c +++ b/drivers/watchdog/qcom-wdt.c @@ -223,10 +223,8 @@ static int qcom_wdt_probe(struct platform_device *pdev) watchdog_init_timeout(&wdt->wdd, 0, dev); ret = devm_watchdog_register_device(dev, &wdt->wdd); - if (ret) { - dev_err(dev, "failed to register watchdog\n"); + if (ret) return ret; - } platform_set_drvdata(pdev, wdt); return 0; diff --git a/drivers/watchdog/rave-sp-wdt.c b/drivers/watchdog/rave-sp-wdt.c index 35db173252f9..2c95615b6354 100644 --- a/drivers/watchdog/rave-sp-wdt.c +++ b/drivers/watchdog/rave-sp-wdt.c @@ -310,7 +310,6 @@ static int rave_sp_wdt_probe(struct platform_device *pdev) ret = devm_watchdog_register_device(dev, wdd); if (ret) { - dev_err(dev, "Failed to register watchdog device\n"); rave_sp_wdt_stop(wdd); return ret; } diff --git a/drivers/watchdog/renesas_wdt.c b/drivers/watchdog/renesas_wdt.c index 565dbc1ec638..00662a8e039c 100644 --- a/drivers/watchdog/renesas_wdt.c +++ b/drivers/watchdog/renesas_wdt.c @@ -7,6 +7,7 @@ */ #include <linux/bitops.h> #include <linux/clk.h> +#include <linux/delay.h> #include <linux/io.h> #include <linux/kernel.h> #include <linux/module.h> @@ -70,6 +71,15 @@ static int rwdt_init_timeout(struct watchdog_device *wdev) return 0; } +static void rwdt_wait_cycles(struct rwdt_priv *priv, unsigned int cycles) +{ + unsigned int delay; + + delay = DIV_ROUND_UP(cycles * 1000000, priv->clk_rate); + + usleep_range(delay, 2 * delay); +} + static int rwdt_start(struct watchdog_device *wdev) { struct rwdt_priv *priv = watchdog_get_drvdata(wdev); @@ -80,6 +90,8 @@ static int rwdt_start(struct watchdog_device *wdev) /* Stop the timer before we modify any register */ val = readb_relaxed(priv->base + RWTCSRA) & ~RWTCSRA_TME; rwdt_write(priv, val, RWTCSRA); + /* Delay 2 cycles before setting watchdog counter */ + rwdt_wait_cycles(priv, 2); rwdt_init_timeout(wdev); rwdt_write(priv, priv->cks, RWTCSRA); @@ -98,6 +110,8 @@ static int rwdt_stop(struct watchdog_device *wdev) struct rwdt_priv *priv = watchdog_get_drvdata(wdev); rwdt_write(priv, priv->cks, RWTCSRA); + /* Delay 3 cycles before disabling module clock */ + rwdt_wait_cycles(priv, 3); pm_runtime_put(wdev->parent); return 0; @@ -175,15 +189,16 @@ static inline bool rwdt_blacklisted(struct device *dev) { return false; } static int rwdt_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct rwdt_priv *priv; struct clk *clk; unsigned long clks_per_sec; int ret, i; - if (rwdt_blacklisted(&pdev->dev)) + if (rwdt_blacklisted(dev)) return -ENODEV; - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -191,16 +206,16 @@ static int rwdt_probe(struct platform_device *pdev) if (IS_ERR(priv->base)) return PTR_ERR(priv->base); - clk = devm_clk_get(&pdev->dev, NULL); + clk = devm_clk_get(dev, NULL); if (IS_ERR(clk)) return PTR_ERR(clk); - pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + pm_runtime_enable(dev); + pm_runtime_get_sync(dev); priv->clk_rate = clk_get_rate(clk); priv->wdev.bootstatus = (readb_relaxed(priv->base + RWTCSRA) & RWTCSRA_WOVF) ? WDIOF_CARDRESET : 0; - pm_runtime_put(&pdev->dev); + pm_runtime_put(dev); if (!priv->clk_rate) { ret = -ENOENT; @@ -216,14 +231,14 @@ static int rwdt_probe(struct platform_device *pdev) } if (i < 0) { - dev_err(&pdev->dev, "Can't find suitable clock divider\n"); + dev_err(dev, "Can't find suitable clock divider\n"); ret = -ERANGE; goto out_pm_disable; } priv->wdev.info = &rwdt_ident; priv->wdev.ops = &rwdt_ops; - priv->wdev.parent = &pdev->dev; + priv->wdev.parent = dev; priv->wdev.min_timeout = 1; priv->wdev.max_timeout = DIV_BY_CLKS_PER_SEC(priv, 65536); priv->wdev.timeout = min(priv->wdev.max_timeout, RWDT_DEFAULT_TIMEOUT); @@ -235,7 +250,7 @@ static int rwdt_probe(struct platform_device *pdev) watchdog_stop_on_unregister(&priv->wdev); /* This overrides the default timeout only if DT configuration was found */ - watchdog_init_timeout(&priv->wdev, 0, &pdev->dev); + watchdog_init_timeout(&priv->wdev, 0, dev); ret = watchdog_register_device(&priv->wdev); if (ret < 0) @@ -244,7 +259,7 @@ static int rwdt_probe(struct platform_device *pdev) return 0; out_pm_disable: - pm_runtime_disable(&pdev->dev); + pm_runtime_disable(dev); return ret; } diff --git a/drivers/watchdog/retu_wdt.c b/drivers/watchdog/retu_wdt.c index 39cd51df2ffc..258dfcf9cbda 100644 --- a/drivers/watchdog/retu_wdt.c +++ b/drivers/watchdog/retu_wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Retu watchdog driver * @@ -5,15 +6,6 @@ * * Based on code written by Amit Kucheria and Michael Buesch. * Rewritten by Aaro Koskinen. - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file "COPYING" in the main directory of this - * archive for more details. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/slab.h> diff --git a/drivers/watchdog/s3c2410_wdt.c b/drivers/watchdog/s3c2410_wdt.c index daf3bf0d86b8..2395f353e52d 100644 --- a/drivers/watchdog/s3c2410_wdt.c +++ b/drivers/watchdog/s3c2410_wdt.c @@ -606,10 +606,8 @@ static int s3c2410wdt_probe(struct platform_device *pdev) wdt->wdt_device.parent = dev; ret = watchdog_register_device(&wdt->wdt_device); - if (ret) { - dev_err(dev, "cannot register watchdog (%d)\n", ret); + if (ret) goto err_cpufreq; - } ret = s3c2410wdt_mask_and_disable_reset(wdt, false); if (ret < 0) diff --git a/drivers/watchdog/sa1100_wdt.c b/drivers/watchdog/sa1100_wdt.c index bfa035e1a75e..cbd8c957182f 100644 --- a/drivers/watchdog/sa1100_wdt.c +++ b/drivers/watchdog/sa1100_wdt.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Watchdog driver for the SA11x0/PXA2xx * * (c) Copyright 2000 Oleg Drokin <green@crimea.edu> * Based on SoftDog driver by Alan Cox <alan@lxorguk.ukuu.org.uk> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Oleg Drokin nor iXcelerator.com admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/sama5d4_wdt.c b/drivers/watchdog/sama5d4_wdt.c index b8da1bf21e12..d193a60430b2 100644 --- a/drivers/watchdog/sama5d4_wdt.c +++ b/drivers/watchdog/sama5d4_wdt.c @@ -110,9 +110,7 @@ static int sama5d4_wdt_set_timeout(struct watchdog_device *wdd, u32 value = WDT_SEC2TICKS(timeout); wdt->mr &= ~AT91_WDT_WDV; - wdt->mr &= ~AT91_WDT_WDD; wdt->mr |= AT91_WDT_SET_WDV(value); - wdt->mr |= AT91_WDT_SET_WDD(value); /* * WDDIS has to be 0 when updating WDD/WDV. The datasheet states: When @@ -248,7 +246,7 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) timeout = WDT_SEC2TICKS(wdd->timeout); - wdt->mr |= AT91_WDT_SET_WDD(timeout); + wdt->mr |= AT91_WDT_SET_WDD(WDT_SEC2TICKS(MAX_WDT_TIMEOUT)); wdt->mr |= AT91_WDT_SET_WDV(timeout); ret = sama5d4_wdt_init(wdt); @@ -259,10 +257,8 @@ static int sama5d4_wdt_probe(struct platform_device *pdev) watchdog_stop_on_unregister(wdd); ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "failed to register watchdog device\n"); + if (ret) return ret; - } platform_set_drvdata(pdev, wdt); @@ -279,7 +275,17 @@ static const struct of_device_id sama5d4_wdt_of_match[] = { MODULE_DEVICE_TABLE(of, sama5d4_wdt_of_match); #ifdef CONFIG_PM_SLEEP -static int sama5d4_wdt_resume(struct device *dev) +static int sama5d4_wdt_suspend_late(struct device *dev) +{ + struct sama5d4_wdt *wdt = dev_get_drvdata(dev); + + if (watchdog_active(&wdt->wdd)) + sama5d4_wdt_stop(&wdt->wdd); + + return 0; +} + +static int sama5d4_wdt_resume_early(struct device *dev) { struct sama5d4_wdt *wdt = dev_get_drvdata(dev); @@ -290,12 +296,17 @@ static int sama5d4_wdt_resume(struct device *dev) */ sama5d4_wdt_init(wdt); + if (watchdog_active(&wdt->wdd)) + sama5d4_wdt_start(&wdt->wdd); + return 0; } #endif -static SIMPLE_DEV_PM_OPS(sama5d4_wdt_pm_ops, NULL, - sama5d4_wdt_resume); +static const struct dev_pm_ops sama5d4_wdt_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(sama5d4_wdt_suspend_late, + sama5d4_wdt_resume_early) +}; static struct platform_driver sama5d4_wdt_driver = { .probe = sama5d4_wdt_probe, diff --git a/drivers/watchdog/sbc7240_wdt.c b/drivers/watchdog/sbc7240_wdt.c index efc81b318939..12cdee7d5069 100644 --- a/drivers/watchdog/sbc7240_wdt.c +++ b/drivers/watchdog/sbc7240_wdt.c @@ -1,19 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * NANO7240 SBC Watchdog device driver * * Based on w83877f.c by Scott Jennings, * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation; - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or - * implied. See the License for the specific language governing - * rights and limitations under the License. - * * (c) Copyright 2007 Gilles GIGAN <gilles.gigan@jcu.edu.au> - * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/watchdog/sbc8360.c b/drivers/watchdog/sbc8360.c index 3396024e7b76..4f8b9912fc51 100644 --- a/drivers/watchdog/sbc8360.c +++ b/drivers/watchdog/sbc8360.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * SBC8360 Watchdog driver * @@ -19,11 +20,6 @@ * (c) Copyright 1996 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/sch311x_wdt.c b/drivers/watchdog/sch311x_wdt.c index ed6e9fac5d74..3612f1df381b 100644 --- a/drivers/watchdog/sch311x_wdt.c +++ b/drivers/watchdog/sch311x_wdt.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * sch311x_wdt.c - Driver for the SCH311x Super-I/O chips * integrated watchdog. * * (c) Copyright 2008 Wim Van Sebroeck <wim@iguana.be>. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Wim Van Sebroeck nor Iguana vzw. admit liability nor * provide warranty for any of this software. This material is * provided "AS-IS" and at no charge. diff --git a/drivers/watchdog/softdog.c b/drivers/watchdog/softdog.c index 060740625485..3e4885c1545e 100644 --- a/drivers/watchdog/softdog.c +++ b/drivers/watchdog/softdog.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * SoftDog: A Software Watchdog Device * * (c) Copyright 1996 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c index cd4430ff9b1c..93bd302ae7c5 100644 --- a/drivers/watchdog/sp5100_tco.c +++ b/drivers/watchdog/sp5100_tco.c @@ -402,10 +402,8 @@ static int sp5100_tco_probe(struct platform_device *pdev) return ret; ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "cannot register watchdog device (err=%d)\n", ret); + if (ret) return ret; - } /* Show module parameters */ dev_info(dev, "initialized. heartbeat=%d sec (nowayout=%d)\n", diff --git a/drivers/watchdog/sp805_wdt.c b/drivers/watchdog/sp805_wdt.c index 072986d461b7..53e04926a7b2 100644 --- a/drivers/watchdog/sp805_wdt.c +++ b/drivers/watchdog/sp805_wdt.c @@ -288,11 +288,8 @@ sp805_wdt_probe(struct amba_device *adev, const struct amba_id *id) } ret = watchdog_register_device(&wdt->wdd); - if (ret) { - dev_err(&adev->dev, "watchdog_register_device() failed: %d\n", - ret); + if (ret) goto err; - } amba_set_drvdata(adev, wdt); dev_info(&adev->dev, "registration successful\n"); diff --git a/drivers/watchdog/sprd_wdt.c b/drivers/watchdog/sprd_wdt.c index 916fb3f96bdc..edba4e278685 100644 --- a/drivers/watchdog/sprd_wdt.c +++ b/drivers/watchdog/sprd_wdt.c @@ -320,7 +320,6 @@ static int sprd_wdt_probe(struct platform_device *pdev) ret = devm_watchdog_register_device(dev, &wdt->wdd); if (ret) { sprd_wdt_disable(wdt); - dev_err(dev, "failed to register watchdog\n"); return ret; } platform_set_drvdata(pdev, wdt); diff --git a/drivers/watchdog/st_lpc_wdt.c b/drivers/watchdog/st_lpc_wdt.c index 7a90184eb950..14ab6559c748 100644 --- a/drivers/watchdog/st_lpc_wdt.c +++ b/drivers/watchdog/st_lpc_wdt.c @@ -228,10 +228,8 @@ static int st_wdog_probe(struct platform_device *pdev) return ret; ret = devm_watchdog_register_device(dev, &st_wdog_dev); - if (ret) { - dev_err(dev, "Unable to register watchdog\n"); + if (ret) return ret; - } st_wdog_setup(st_wdog, true); diff --git a/drivers/watchdog/stm32_iwdg.c b/drivers/watchdog/stm32_iwdg.c index d569a3634d9b..a3a329011a06 100644 --- a/drivers/watchdog/stm32_iwdg.c +++ b/drivers/watchdog/stm32_iwdg.c @@ -263,10 +263,8 @@ static int stm32_iwdg_probe(struct platform_device *pdev) watchdog_init_timeout(wdd, 0, dev); ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "failed to register watchdog device\n"); + if (ret) return ret; - } platform_set_drvdata(pdev, wdt); diff --git a/drivers/watchdog/stmp3xxx_rtc_wdt.c b/drivers/watchdog/stmp3xxx_rtc_wdt.c index 671f4ba7b4ed..7caf3aa71c6a 100644 --- a/drivers/watchdog/stmp3xxx_rtc_wdt.c +++ b/drivers/watchdog/stmp3xxx_rtc_wdt.c @@ -98,10 +98,8 @@ static int stmp3xxx_wdt_probe(struct platform_device *pdev) stmp3xxx_wdd.parent = dev; ret = devm_watchdog_register_device(dev, &stmp3xxx_wdd); - if (ret < 0) { - dev_err(dev, "cannot register watchdog device\n"); + if (ret < 0) return ret; - } if (register_reboot_notifier(&wdt_notifier)) dev_warn(dev, "cannot register reboot notifier\n"); diff --git a/drivers/watchdog/tegra_wdt.c b/drivers/watchdog/tegra_wdt.c index a58b000acc4f..dfe06e506cad 100644 --- a/drivers/watchdog/tegra_wdt.c +++ b/drivers/watchdog/tegra_wdt.c @@ -219,10 +219,8 @@ static int tegra_wdt_probe(struct platform_device *pdev) watchdog_stop_on_unregister(wdd); ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "failed to register watchdog device\n"); + if (ret) return ret; - } platform_set_drvdata(pdev, wdt); diff --git a/drivers/watchdog/ts4800_wdt.c b/drivers/watchdog/ts4800_wdt.c index 9dc6d7f45806..c137ad2bd5c3 100644 --- a/drivers/watchdog/ts4800_wdt.c +++ b/drivers/watchdog/ts4800_wdt.c @@ -171,10 +171,8 @@ static int ts4800_wdt_probe(struct platform_device *pdev) ts4800_wdt_stop(wdd); ret = devm_watchdog_register_device(dev, wdd); - if (ret) { - dev_err(dev, "failed to register watchdog device\n"); + if (ret) return ret; - } platform_set_drvdata(pdev, wdt); diff --git a/drivers/watchdog/w83627hf_wdt.c b/drivers/watchdog/w83627hf_wdt.c index 3a49ba9ea608..38b31e9947aa 100644 --- a/drivers/watchdog/w83627hf_wdt.c +++ b/drivers/watchdog/w83627hf_wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * w83627hf/thf WDT driver * @@ -17,11 +18,6 @@ * (c) Copyright 1996 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/wafer5823wdt.c b/drivers/watchdog/wafer5823wdt.c index 0a8073b419f8..6d2071a0590d 100644 --- a/drivers/watchdog/wafer5823wdt.c +++ b/drivers/watchdog/wafer5823wdt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * ICP Wafer 5823 Single Board Computer WDT driver * http://www.icpamerica.com/wafer_5823.php @@ -13,11 +14,6 @@ * (c) Copyright 1996-1997 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/watchdog_core.c b/drivers/watchdog/watchdog_core.c index 62be9e52a4de..21e8085b848b 100644 --- a/drivers/watchdog/watchdog_core.c +++ b/drivers/watchdog/watchdog_core.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * watchdog_core.c * @@ -16,11 +17,6 @@ * Satyam Sharma <satyam@infradead.org> * Randy Dunlap <randy.dunlap@oracle.com> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox, CymruNet Ltd., Wim Van Sebroeck nor Iguana vzw. * admit liability nor provide warranty for any of this software. * This material is provided "AS-IS" and at no charge. @@ -60,11 +56,10 @@ static DEFINE_MUTEX(wtd_deferred_reg_mutex); static LIST_HEAD(wtd_deferred_reg_list); static bool wtd_deferred_reg_done; -static int watchdog_deferred_registration_add(struct watchdog_device *wdd) +static void watchdog_deferred_registration_add(struct watchdog_device *wdd) { list_add_tail(&wdd->deferred, &wtd_deferred_reg_list); - return 0; } static void watchdog_deferred_registration_del(struct watchdog_device *wdd) @@ -265,14 +260,23 @@ static int __watchdog_register_device(struct watchdog_device *wdd) int watchdog_register_device(struct watchdog_device *wdd) { - int ret; + const char *dev_str; + int ret = 0; mutex_lock(&wtd_deferred_reg_mutex); if (wtd_deferred_reg_done) ret = __watchdog_register_device(wdd); else - ret = watchdog_deferred_registration_add(wdd); + watchdog_deferred_registration_add(wdd); mutex_unlock(&wtd_deferred_reg_mutex); + + if (ret) { + dev_str = wdd->parent ? dev_name(wdd->parent) : + (const char *)wdd->info->identity; + pr_err("%s: failed to register watchdog device (err = %d)\n", + dev_str, ret); + } + return ret; } EXPORT_SYMBOL_GPL(watchdog_register_device); diff --git a/drivers/watchdog/watchdog_core.h b/drivers/watchdog/watchdog_core.h index 86ff962d1e15..a5062e8e0d13 100644 --- a/drivers/watchdog/watchdog_core.h +++ b/drivers/watchdog/watchdog_core.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ /* * watchdog_core.h * @@ -16,11 +17,6 @@ * Satyam Sharma <satyam@infradead.org> * Randy Dunlap <randy.dunlap@oracle.com> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox, CymruNet Ltd., Wim Van Sebroeck nor Iguana vzw. * admit liability nor provide warranty for any of this software. * This material is provided "AS-IS" and at no charge. diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c index 252a7c7b6592..dbd2ad4c9294 100644 --- a/drivers/watchdog/watchdog_dev.c +++ b/drivers/watchdog/watchdog_dev.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * watchdog_dev.c * @@ -20,11 +21,6 @@ * Satyam Sharma <satyam@infradead.org> * Randy Dunlap <randy.dunlap@oracle.com> * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox, CymruNet Ltd., Wim Van Sebroeck nor Iguana vzw. * admit liability nor provide warranty for any of this software. * This material is provided "AS-IS" and at no charge. @@ -69,6 +65,7 @@ struct watchdog_core_data { struct mutex lock; ktime_t last_keepalive; ktime_t last_hw_keepalive; + ktime_t open_deadline; struct hrtimer timer; struct kthread_work work; unsigned long status; /* Internal status bits */ @@ -87,6 +84,19 @@ static struct kthread_worker *watchdog_kworker; static bool handle_boot_enabled = IS_ENABLED(CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED); +static unsigned open_timeout = CONFIG_WATCHDOG_OPEN_TIMEOUT; + +static bool watchdog_past_open_deadline(struct watchdog_core_data *data) +{ + return ktime_after(ktime_get(), data->open_deadline); +} + +static void watchdog_set_open_deadline(struct watchdog_core_data *data) +{ + data->open_deadline = open_timeout ? + ktime_get() + ktime_set(open_timeout, 0) : KTIME_MAX; +} + static inline bool watchdog_need_worker(struct watchdog_device *wdd) { /* All variables in milli-seconds */ @@ -119,14 +129,15 @@ static ktime_t watchdog_next_keepalive(struct watchdog_device *wdd) ktime_t virt_timeout; unsigned int hw_heartbeat_ms; - virt_timeout = ktime_add(wd_data->last_keepalive, - ms_to_ktime(timeout_ms)); + if (watchdog_active(wdd)) + virt_timeout = ktime_add(wd_data->last_keepalive, + ms_to_ktime(timeout_ms)); + else + virt_timeout = wd_data->open_deadline; + hw_heartbeat_ms = min_not_zero(timeout_ms, wdd->max_hw_heartbeat_ms); keepalive_interval = ms_to_ktime(hw_heartbeat_ms / 2); - if (!watchdog_active(wdd)) - return keepalive_interval; - /* * To ensure that the watchdog times out wdd->timeout seconds * after the most recent ping from userspace, the last @@ -211,7 +222,13 @@ static bool watchdog_worker_should_ping(struct watchdog_core_data *wd_data) { struct watchdog_device *wdd = wd_data->wdd; - return wdd && (watchdog_active(wdd) || watchdog_hw_running(wdd)); + if (!wdd) + return false; + + if (watchdog_active(wdd)) + return true; + + return watchdog_hw_running(wdd) && !watchdog_past_open_deadline(wd_data); } static void watchdog_ping_work(struct kthread_work *work) @@ -824,6 +841,15 @@ static int watchdog_open(struct inode *inode, struct file *file) if (!hw_running) kref_get(&wd_data->kref); + /* + * open_timeout only applies for the first open from + * userspace. Set open_deadline to infinity so that the kernel + * will take care of an always-running hardware watchdog in + * case the device gets magic-closed or WDIOS_DISABLECARD is + * applied. + */ + wd_data->open_deadline = KTIME_MAX; + /* dev/watchdog is a virtual (and thus non-seekable) filesystem */ return stream_open(inode, file); @@ -983,6 +1009,7 @@ static int watchdog_cdev_register(struct watchdog_device *wdd, dev_t devno) /* Record time of most recent heartbeat as 'just before now'. */ wd_data->last_hw_keepalive = ktime_sub(ktime_get(), 1); + watchdog_set_open_deadline(wd_data); /* * If the watchdog is running, prevent its driver from being unloaded, @@ -1181,3 +1208,8 @@ module_param(handle_boot_enabled, bool, 0444); MODULE_PARM_DESC(handle_boot_enabled, "Watchdog core auto-updates boot enabled watchdogs before userspace takes over (default=" __MODULE_STRING(IS_ENABLED(CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED)) ")"); + +module_param(open_timeout, uint, 0644); +MODULE_PARM_DESC(open_timeout, + "Maximum time (in seconds, 0 means infinity) for userspace to take over a running watchdog (default=" + __MODULE_STRING(CONFIG_WATCHDOG_OPEN_TIMEOUT) ")"); diff --git a/drivers/watchdog/wd501p.h b/drivers/watchdog/wd501p.h index 0e3a497d5626..43a4d88fd363 100644 --- a/drivers/watchdog/wd501p.h +++ b/drivers/watchdog/wd501p.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-1.0+ */ /* * Industrial Computer Source WDT500/501 driver * @@ -11,12 +12,7 @@ * * http://www.cymru.net * - * This driver is provided under the GNU General Public License, - * incorporated herein by reference. The driver is provided without - * warranty or support. - * * Release 0.04. - * */ diff --git a/drivers/watchdog/wdt.c b/drivers/watchdog/wdt.c index 3d2f5ed60e88..0650100fad00 100644 --- a/drivers/watchdog/wdt.c +++ b/drivers/watchdog/wdt.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Industrial Computer Source WDT501 driver * * (c) Copyright 1996-1997 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/wdt_pci.c b/drivers/watchdog/wdt_pci.c index ff3a41f47127..66303ab95685 100644 --- a/drivers/watchdog/wdt_pci.c +++ b/drivers/watchdog/wdt_pci.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Industrial Computer Source PCI-WDT500/501 driver * * (c) Copyright 1996-1997 Alan Cox <alan@lxorguk.ukuu.org.uk>, * All Rights Reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Neither Alan Cox nor CymruNet Ltd. admit liability nor provide * warranty for any of this software. This material is provided * "AS-IS" and at no charge. diff --git a/drivers/watchdog/wm831x_wdt.c b/drivers/watchdog/wm831x_wdt.c index 9b6565a3fab4..030ce240620d 100644 --- a/drivers/watchdog/wm831x_wdt.c +++ b/drivers/watchdog/wm831x_wdt.c @@ -267,14 +267,7 @@ static int wm831x_wdt_probe(struct platform_device *pdev) } } - ret = devm_watchdog_register_device(dev, &driver_data->wdt); - if (ret != 0) { - dev_err(wm831x->dev, "watchdog_register_device() failed: %d\n", - ret); - return ret; - } - - return 0; + return devm_watchdog_register_device(dev, &driver_data->wdt); } static struct platform_driver wm831x_wdt_driver = { diff --git a/drivers/watchdog/xen_wdt.c b/drivers/watchdog/xen_wdt.c index 2ba0a3c4523c..b343f421dc72 100644 --- a/drivers/watchdog/xen_wdt.c +++ b/drivers/watchdog/xen_wdt.c @@ -138,10 +138,8 @@ static int xen_wdt_probe(struct platform_device *pdev) watchdog_stop_on_unregister(&xen_wdt_dev); ret = devm_watchdog_register_device(dev, &xen_wdt_dev); - if (ret) { - dev_err(dev, "cannot register watchdog device (%d)\n", ret); + if (ret) return ret; - } dev_info(dev, "initialized (timeout=%ds, nowayout=%d)\n", xen_wdt_dev.timeout, nowayout); diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index d53f3493a6b9..cfbe46785a3b 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -402,7 +402,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir, attrs); - if (map == DMA_MAPPING_ERROR) + if (map == (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; dev_addr = xen_phys_to_bus(map); @@ -2094,7 +2094,6 @@ SYSCALL_DEFINE6(io_pgetevents, const struct __aio_sigset __user *, usig) { struct __aio_sigset ksig = { NULL, }; - sigset_t ksigmask, sigsaved; struct timespec64 ts; bool interrupted; int ret; @@ -2105,14 +2104,14 @@ SYSCALL_DEFINE6(io_pgetevents, if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) return -EFAULT; - ret = set_user_sigmask(ksig.sigmask, &ksigmask, &sigsaved, ksig.sigsetsize); + ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); if (ret) return ret; ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); interrupted = signal_pending(current); - restore_user_sigmask(ksig.sigmask, &sigsaved, interrupted); + restore_saved_sigmask_unless(interrupted); if (interrupted && !ret) ret = -ERESTARTNOHAND; @@ -2130,7 +2129,6 @@ SYSCALL_DEFINE6(io_pgetevents_time32, const struct __aio_sigset __user *, usig) { struct __aio_sigset ksig = { NULL, }; - sigset_t ksigmask, sigsaved; struct timespec64 ts; bool interrupted; int ret; @@ -2142,14 +2140,14 @@ SYSCALL_DEFINE6(io_pgetevents_time32, return -EFAULT; - ret = set_user_sigmask(ksig.sigmask, &ksigmask, &sigsaved, ksig.sigsetsize); + ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); if (ret) return ret; ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); interrupted = signal_pending(current); - restore_user_sigmask(ksig.sigmask, &sigsaved, interrupted); + restore_saved_sigmask_unless(interrupted); if (interrupted && !ret) ret = -ERESTARTNOHAND; @@ -2198,7 +2196,6 @@ COMPAT_SYSCALL_DEFINE6(io_pgetevents, const struct __compat_aio_sigset __user *, usig) { struct __compat_aio_sigset ksig = { NULL, }; - sigset_t ksigmask, sigsaved; struct timespec64 t; bool interrupted; int ret; @@ -2209,14 +2206,14 @@ COMPAT_SYSCALL_DEFINE6(io_pgetevents, if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) return -EFAULT; - ret = set_compat_user_sigmask(ksig.sigmask, &ksigmask, &sigsaved, ksig.sigsetsize); + ret = set_compat_user_sigmask(ksig.sigmask, ksig.sigsetsize); if (ret) return ret; ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); interrupted = signal_pending(current); - restore_user_sigmask(ksig.sigmask, &sigsaved, interrupted); + restore_saved_sigmask_unless(interrupted); if (interrupted && !ret) ret = -ERESTARTNOHAND; @@ -2234,7 +2231,6 @@ COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64, const struct __compat_aio_sigset __user *, usig) { struct __compat_aio_sigset ksig = { NULL, }; - sigset_t ksigmask, sigsaved; struct timespec64 t; bool interrupted; int ret; @@ -2245,14 +2241,14 @@ COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64, if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) return -EFAULT; - ret = set_compat_user_sigmask(ksig.sigmask, &ksigmask, &sigsaved, ksig.sigsetsize); + ret = set_compat_user_sigmask(ksig.sigmask, ksig.sigsetsize); if (ret) return ret; ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); interrupted = signal_pending(current); - restore_user_sigmask(ksig.sigmask, &sigsaved, interrupted); + restore_saved_sigmask_unless(interrupted); if (interrupted && !ret) ret = -ERESTARTNOHAND; diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 8264b468f283..d4e11b2e04f6 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1127,7 +1127,6 @@ out_free_interp: load_addr, interp_load_addr); if (retval < 0) goto out; - /* N.B. passed_fileno might not be initialized? */ current->mm->end_code = end_code; current->mm->start_code = start_code; current->mm->start_data = start_data; diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c index 8c6b50f34466..831a2b25ba79 100644 --- a/fs/binfmt_flat.c +++ b/fs/binfmt_flat.c @@ -431,7 +431,6 @@ static int load_flat_file(struct linux_binprm *bprm, unsigned long len, memp, memp_size, extra, rlim; __be32 __user *reloc; u32 __user *rp; - struct inode *inode; int i, rev, relocs; loff_t fpos; unsigned long start_code, end_code; @@ -439,7 +438,6 @@ static int load_flat_file(struct linux_binprm *bprm, int ret; hdr = ((struct flat_hdr *) bprm->buf); /* exec-header */ - inode = file_inode(bprm->file); text_len = ntohl(hdr->data_start); data_len = ntohl(hdr->data_end) - ntohl(hdr->data_start); diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 23537bc8c827..212b4a854f2c 100644 --- a/fs/btrfs/Kconfig +++ b/fs/btrfs/Kconfig @@ -2,7 +2,8 @@ config BTRFS_FS tristate "Btrfs filesystem support" - select LIBCRC32C + select CRYPTO + select CRYPTO_CRC32C select ZLIB_INFLATE select ZLIB_DEFLATE select LZO_COMPRESS diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile index ca693dd554e9..76a843198bcb 100644 --- a/fs/btrfs/Makefile +++ b/fs/btrfs/Makefile @@ -10,7 +10,8 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ export.o tree-log.o free-space-cache.o zlib.o lzo.o zstd.o \ compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \ reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ - uuid-tree.o props.o free-space-tree.o tree-checker.o + uuid-tree.o props.o free-space-tree.o tree-checker.o space-info.o \ + block-rsv.o delalloc-space.o btrfs-$(CONFIG_BTRFS_FS_POSIX_ACL) += acl.o btrfs-$(CONFIG_BTRFS_FS_CHECK_INTEGRITY) += check-integrity.o diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 982152d3f920..89116afda7a2 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1465,12 +1465,11 @@ int btrfs_find_all_roots(struct btrfs_trans_handle *trans, * * Return: 0 if extent is not shared, 1 if it is shared, < 0 on error. */ -int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr) +int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, + struct ulist *roots, struct ulist *tmp) { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_trans_handle *trans; - struct ulist *tmp = NULL; - struct ulist *roots = NULL; struct ulist_iterator uiter; struct ulist_node *node; struct seq_list elem = SEQ_LIST_INIT(elem); @@ -1481,12 +1480,8 @@ int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr) .share_count = 0, }; - tmp = ulist_alloc(GFP_NOFS); - roots = ulist_alloc(GFP_NOFS); - if (!tmp || !roots) { - ret = -ENOMEM; - goto out; - } + ulist_init(roots); + ulist_init(tmp); trans = btrfs_attach_transaction(root); if (IS_ERR(trans)) { @@ -1527,8 +1522,8 @@ int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr) up_read(&fs_info->commit_root_sem); } out: - ulist_free(tmp); - ulist_free(roots); + ulist_release(roots); + ulist_release(tmp); return ret; } diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 54d58988483a..777f61dc081e 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -57,7 +57,8 @@ int btrfs_find_one_extref(struct btrfs_root *root, u64 inode_objectid, u64 start_off, struct btrfs_path *path, struct btrfs_inode_extref **ret_extref, u64 *found_off); -int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr); +int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, + struct ulist *roots, struct ulist *tmp_ulist); int __init btrfs_prelim_ref_init(void); void __cold btrfs_prelim_ref_exit(void); diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c new file mode 100644 index 000000000000..698470b9f32d --- /dev/null +++ b/fs/btrfs/block-rsv.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "ctree.h" +#include "block-rsv.h" +#include "space-info.h" +#include "math.h" +#include "transaction.h" + +static u64 block_rsv_release_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + struct btrfs_block_rsv *dest, u64 num_bytes, + u64 *qgroup_to_release_ret) +{ + struct btrfs_space_info *space_info = block_rsv->space_info; + u64 qgroup_to_release = 0; + u64 ret; + + spin_lock(&block_rsv->lock); + if (num_bytes == (u64)-1) { + num_bytes = block_rsv->size; + qgroup_to_release = block_rsv->qgroup_rsv_size; + } + block_rsv->size -= num_bytes; + if (block_rsv->reserved >= block_rsv->size) { + num_bytes = block_rsv->reserved - block_rsv->size; + block_rsv->reserved = block_rsv->size; + block_rsv->full = 1; + } else { + num_bytes = 0; + } + if (block_rsv->qgroup_rsv_reserved >= block_rsv->qgroup_rsv_size) { + qgroup_to_release = block_rsv->qgroup_rsv_reserved - + block_rsv->qgroup_rsv_size; + block_rsv->qgroup_rsv_reserved = block_rsv->qgroup_rsv_size; + } else { + qgroup_to_release = 0; + } + spin_unlock(&block_rsv->lock); + + ret = num_bytes; + if (num_bytes > 0) { + if (dest) { + spin_lock(&dest->lock); + if (!dest->full) { + u64 bytes_to_add; + + bytes_to_add = dest->size - dest->reserved; + bytes_to_add = min(num_bytes, bytes_to_add); + dest->reserved += bytes_to_add; + if (dest->reserved >= dest->size) + dest->full = 1; + num_bytes -= bytes_to_add; + } + spin_unlock(&dest->lock); + } + if (num_bytes) + btrfs_space_info_add_old_bytes(fs_info, space_info, + num_bytes); + } + if (qgroup_to_release_ret) + *qgroup_to_release_ret = qgroup_to_release; + return ret; +} + +int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src, + struct btrfs_block_rsv *dst, u64 num_bytes, + bool update_size) +{ + int ret; + + ret = btrfs_block_rsv_use_bytes(src, num_bytes); + if (ret) + return ret; + + btrfs_block_rsv_add_bytes(dst, num_bytes, update_size); + return 0; +} + +void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type) +{ + memset(rsv, 0, sizeof(*rsv)); + spin_lock_init(&rsv->lock); + rsv->type = type; +} + +void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv, + unsigned short type) +{ + btrfs_init_block_rsv(rsv, type); + rsv->space_info = btrfs_find_space_info(fs_info, + BTRFS_BLOCK_GROUP_METADATA); +} + +struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, + unsigned short type) +{ + struct btrfs_block_rsv *block_rsv; + + block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS); + if (!block_rsv) + return NULL; + + btrfs_init_metadata_block_rsv(fs_info, block_rsv, type); + return block_rsv; +} + +void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv) +{ + if (!rsv) + return; + btrfs_block_rsv_release(fs_info, rsv, (u64)-1); + kfree(rsv); +} + +int btrfs_block_rsv_add(struct btrfs_root *root, + struct btrfs_block_rsv *block_rsv, u64 num_bytes, + enum btrfs_reserve_flush_enum flush) +{ + int ret; + + if (num_bytes == 0) + return 0; + + ret = btrfs_reserve_metadata_bytes(root, block_rsv, num_bytes, flush); + if (!ret) + btrfs_block_rsv_add_bytes(block_rsv, num_bytes, true); + + return ret; +} + +int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor) +{ + u64 num_bytes = 0; + int ret = -ENOSPC; + + if (!block_rsv) + return 0; + + spin_lock(&block_rsv->lock); + num_bytes = div_factor(block_rsv->size, min_factor); + if (block_rsv->reserved >= num_bytes) + ret = 0; + spin_unlock(&block_rsv->lock); + + return ret; +} + +int btrfs_block_rsv_refill(struct btrfs_root *root, + struct btrfs_block_rsv *block_rsv, u64 min_reserved, + enum btrfs_reserve_flush_enum flush) +{ + u64 num_bytes = 0; + int ret = -ENOSPC; + + if (!block_rsv) + return 0; + + spin_lock(&block_rsv->lock); + num_bytes = min_reserved; + if (block_rsv->reserved >= num_bytes) + ret = 0; + else + num_bytes -= block_rsv->reserved; + spin_unlock(&block_rsv->lock); + + if (!ret) + return 0; + + ret = btrfs_reserve_metadata_bytes(root, block_rsv, num_bytes, flush); + if (!ret) { + btrfs_block_rsv_add_bytes(block_rsv, num_bytes, false); + return 0; + } + + return ret; +} + +u64 __btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + u64 num_bytes, u64 *qgroup_to_release) +{ + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_block_rsv *target = NULL; + + /* + * If we are the delayed_rsv then push to the global rsv, otherwise dump + * into the delayed rsv if it is not full. + */ + if (block_rsv == delayed_rsv) + target = global_rsv; + else if (block_rsv != global_rsv && !delayed_rsv->full) + target = delayed_rsv; + + if (target && block_rsv->space_info != target->space_info) + target = NULL; + + return block_rsv_release_bytes(fs_info, block_rsv, target, num_bytes, + qgroup_to_release); +} + +int btrfs_block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, u64 num_bytes) +{ + int ret = -ENOSPC; + + spin_lock(&block_rsv->lock); + if (block_rsv->reserved >= num_bytes) { + block_rsv->reserved -= num_bytes; + if (block_rsv->reserved < block_rsv->size) + block_rsv->full = 0; + ret = 0; + } + spin_unlock(&block_rsv->lock); + return ret; +} + +void btrfs_block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, + u64 num_bytes, bool update_size) +{ + spin_lock(&block_rsv->lock); + block_rsv->reserved += num_bytes; + if (update_size) + block_rsv->size += num_bytes; + else if (block_rsv->reserved >= block_rsv->size) + block_rsv->full = 1; + spin_unlock(&block_rsv->lock); +} + +int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *dest, u64 num_bytes, + int min_factor) +{ + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + u64 min_bytes; + + if (global_rsv->space_info != dest->space_info) + return -ENOSPC; + + spin_lock(&global_rsv->lock); + min_bytes = div_factor(global_rsv->size, min_factor); + if (global_rsv->reserved < min_bytes + num_bytes) { + spin_unlock(&global_rsv->lock); + return -ENOSPC; + } + global_rsv->reserved -= num_bytes; + if (global_rsv->reserved < global_rsv->size) + global_rsv->full = 0; + spin_unlock(&global_rsv->lock); + + btrfs_block_rsv_add_bytes(dest, num_bytes, true); + return 0; +} + +void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; + struct btrfs_space_info *sinfo = block_rsv->space_info; + u64 num_bytes; + + /* + * The global block rsv is based on the size of the extent tree, the + * checksum tree and the root tree. If the fs is empty we want to set + * it to a minimal amount for safety. + */ + num_bytes = btrfs_root_used(&fs_info->extent_root->root_item) + + btrfs_root_used(&fs_info->csum_root->root_item) + + btrfs_root_used(&fs_info->tree_root->root_item); + num_bytes = max_t(u64, num_bytes, SZ_16M); + + spin_lock(&sinfo->lock); + spin_lock(&block_rsv->lock); + + block_rsv->size = min_t(u64, num_bytes, SZ_512M); + + if (block_rsv->reserved < block_rsv->size) { + num_bytes = btrfs_space_info_used(sinfo, true); + if (sinfo->total_bytes > num_bytes) { + num_bytes = sinfo->total_bytes - num_bytes; + num_bytes = min(num_bytes, + block_rsv->size - block_rsv->reserved); + block_rsv->reserved += num_bytes; + btrfs_space_info_update_bytes_may_use(fs_info, sinfo, + num_bytes); + trace_btrfs_space_reservation(fs_info, "space_info", + sinfo->flags, num_bytes, + 1); + } + } else if (block_rsv->reserved > block_rsv->size) { + num_bytes = block_rsv->reserved - block_rsv->size; + btrfs_space_info_update_bytes_may_use(fs_info, sinfo, + -num_bytes); + trace_btrfs_space_reservation(fs_info, "space_info", + sinfo->flags, num_bytes, 0); + block_rsv->reserved = block_rsv->size; + } + + if (block_rsv->reserved == block_rsv->size) + block_rsv->full = 1; + else + block_rsv->full = 0; + + spin_unlock(&block_rsv->lock); + spin_unlock(&sinfo->lock); +} + +void btrfs_init_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + struct btrfs_space_info *space_info; + + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); + fs_info->chunk_block_rsv.space_info = space_info; + + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); + fs_info->global_block_rsv.space_info = space_info; + fs_info->trans_block_rsv.space_info = space_info; + fs_info->empty_block_rsv.space_info = space_info; + fs_info->delayed_block_rsv.space_info = space_info; + fs_info->delayed_refs_rsv.space_info = space_info; + + fs_info->extent_root->block_rsv = &fs_info->delayed_refs_rsv; + fs_info->csum_root->block_rsv = &fs_info->delayed_refs_rsv; + fs_info->dev_root->block_rsv = &fs_info->global_block_rsv; + fs_info->tree_root->block_rsv = &fs_info->global_block_rsv; + if (fs_info->quota_root) + fs_info->quota_root->block_rsv = &fs_info->global_block_rsv; + fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv; + + btrfs_update_global_block_rsv(fs_info); +} + +void btrfs_release_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + btrfs_block_rsv_release(fs_info, &fs_info->global_block_rsv, (u64)-1); + WARN_ON(fs_info->trans_block_rsv.size > 0); + WARN_ON(fs_info->trans_block_rsv.reserved > 0); + WARN_ON(fs_info->chunk_block_rsv.size > 0); + WARN_ON(fs_info->chunk_block_rsv.reserved > 0); + WARN_ON(fs_info->delayed_block_rsv.size > 0); + WARN_ON(fs_info->delayed_block_rsv.reserved > 0); + WARN_ON(fs_info->delayed_refs_rsv.reserved > 0); + WARN_ON(fs_info->delayed_refs_rsv.size > 0); +} + +static struct btrfs_block_rsv *get_block_rsv( + const struct btrfs_trans_handle *trans, + const struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *block_rsv = NULL; + + if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || + (root == fs_info->csum_root && trans->adding_csums) || + (root == fs_info->uuid_root)) + block_rsv = trans->block_rsv; + + if (!block_rsv) + block_rsv = root->block_rsv; + + if (!block_rsv) + block_rsv = &fs_info->empty_block_rsv; + + return block_rsv; +} + +struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u32 blocksize) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *block_rsv; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + int ret; + bool global_updated = false; + + block_rsv = get_block_rsv(trans, root); + + if (unlikely(block_rsv->size == 0)) + goto try_reserve; +again: + ret = btrfs_block_rsv_use_bytes(block_rsv, blocksize); + if (!ret) + return block_rsv; + + if (block_rsv->failfast) + return ERR_PTR(ret); + + if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) { + global_updated = true; + btrfs_update_global_block_rsv(fs_info); + goto again; + } + + /* + * The global reserve still exists to save us from ourselves, so don't + * warn_on if we are short on our delayed refs reserve. + */ + if (block_rsv->type != BTRFS_BLOCK_RSV_DELREFS && + btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL * 10, + /*DEFAULT_RATELIMIT_BURST*/ 1); + if (__ratelimit(&_rs)) + WARN(1, KERN_DEBUG + "BTRFS: block rsv returned %d\n", ret); + } +try_reserve: + ret = btrfs_reserve_metadata_bytes(root, block_rsv, blocksize, + BTRFS_RESERVE_NO_FLUSH); + if (!ret) + return block_rsv; + /* + * If we couldn't reserve metadata bytes try and use some from + * the global reserve if its space type is the same as the global + * reservation. + */ + if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL && + block_rsv->space_info == global_rsv->space_info) { + ret = btrfs_block_rsv_use_bytes(global_rsv, blocksize); + if (!ret) + return global_rsv; + } + return ERR_PTR(ret); +} diff --git a/fs/btrfs/block-rsv.h b/fs/btrfs/block-rsv.h new file mode 100644 index 000000000000..d1428bb73fc5 --- /dev/null +++ b/fs/btrfs/block-rsv.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_BLOCK_RSV_H +#define BTRFS_BLOCK_RSV_H + +struct btrfs_trans_handle; +enum btrfs_reserve_flush_enum; + +/* + * Types of block reserves + */ +enum { + BTRFS_BLOCK_RSV_GLOBAL, + BTRFS_BLOCK_RSV_DELALLOC, + BTRFS_BLOCK_RSV_TRANS, + BTRFS_BLOCK_RSV_CHUNK, + BTRFS_BLOCK_RSV_DELOPS, + BTRFS_BLOCK_RSV_DELREFS, + BTRFS_BLOCK_RSV_EMPTY, + BTRFS_BLOCK_RSV_TEMP, +}; + +struct btrfs_block_rsv { + u64 size; + u64 reserved; + struct btrfs_space_info *space_info; + spinlock_t lock; + unsigned short full; + unsigned short type; + unsigned short failfast; + + /* + * Qgroup equivalent for @size @reserved + * + * Unlike normal @size/@reserved for inode rsv, qgroup doesn't care + * about things like csum size nor how many tree blocks it will need to + * reserve. + * + * Qgroup cares more about net change of the extent usage. + * + * So for one newly inserted file extent, in worst case it will cause + * leaf split and level increase, nodesize for each file extent is + * already too much. + * + * In short, qgroup_size/reserved is the upper limit of possible needed + * qgroup metadata reservation. + */ + u64 qgroup_rsv_size; + u64 qgroup_rsv_reserved; +}; + +void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type); +struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, + unsigned short type); +void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv, + unsigned short type); +void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv); +int btrfs_block_rsv_add(struct btrfs_root *root, + struct btrfs_block_rsv *block_rsv, u64 num_bytes, + enum btrfs_reserve_flush_enum flush); +int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor); +int btrfs_block_rsv_refill(struct btrfs_root *root, + struct btrfs_block_rsv *block_rsv, u64 min_reserved, + enum btrfs_reserve_flush_enum flush); +int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv, + struct btrfs_block_rsv *dst_rsv, u64 num_bytes, + bool update_size); +int btrfs_block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, u64 num_bytes); +int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *dest, u64 num_bytes, + int min_factor); +void btrfs_block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, + u64 num_bytes, bool update_size); +u64 __btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + u64 num_bytes, u64 *qgroup_to_release); +void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info); +void btrfs_init_global_block_rsv(struct btrfs_fs_info *fs_info); +void btrfs_release_global_block_rsv(struct btrfs_fs_info *fs_info); +struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u32 blocksize); + +static inline void btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + u64 num_bytes) +{ + __btrfs_block_rsv_release(fs_info, block_rsv, num_bytes, NULL); +} + +static inline void btrfs_unuse_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + u32 blocksize) +{ + btrfs_block_rsv_add_bytes(block_rsv, blocksize, false); + btrfs_block_rsv_release(fs_info, block_rsv, 0); +} + +#endif /* BTRFS_BLOCK_RSV_H */ diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index d5b438706b77..f853835c409c 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -337,22 +337,34 @@ static inline void btrfs_inode_resume_unlocked_dio(struct btrfs_inode *inode) clear_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags); } +/* Array of bytes with variable length, hexadecimal format 0x1234 */ +#define CSUM_FMT "0x%*phN" +#define CSUM_FMT_VALUE(size, bytes) size, bytes + static inline void btrfs_print_data_csum_error(struct btrfs_inode *inode, - u64 logical_start, u32 csum, u32 csum_expected, int mirror_num) + u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num) { struct btrfs_root *root = inode->root; + struct btrfs_super_block *sb = root->fs_info->super_copy; + const u16 csum_size = btrfs_super_csum_size(sb); /* Output minus objectid, which is more meaningful */ if (root->root_key.objectid >= BTRFS_LAST_FREE_OBJECTID) btrfs_warn_rl(root->fs_info, - "csum failed root %lld ino %lld off %llu csum 0x%08x expected csum 0x%08x mirror %d", +"csum failed root %lld ino %lld off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d", root->root_key.objectid, btrfs_ino(inode), - logical_start, csum, csum_expected, mirror_num); + logical_start, + CSUM_FMT_VALUE(csum_size, csum), + CSUM_FMT_VALUE(csum_size, csum_expected), + mirror_num); else btrfs_warn_rl(root->fs_info, - "csum failed root %llu ino %llu off %llu csum 0x%08x expected csum 0x%08x mirror %d", +"csum failed root %llu ino %llu off %llu csum " CSUM_FMT " expected csum " CSUM_FMT " mirror %d", root->root_key.objectid, btrfs_ino(inode), - logical_start, csum, csum_expected, mirror_num); + logical_start, + CSUM_FMT_VALUE(csum_size, csum), + CSUM_FMT_VALUE(csum_size, csum_expected), + mirror_num); } #endif diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c index b0c8094528d1..81a9731959a9 100644 --- a/fs/btrfs/check-integrity.c +++ b/fs/btrfs/check-integrity.c @@ -83,7 +83,7 @@ #include <linux/blkdev.h> #include <linux/mm.h> #include <linux/string.h> -#include <linux/crc32c.h> +#include <crypto/hash.h> #include "ctree.h" #include "disk-io.h" #include "transaction.h" @@ -1710,9 +1710,9 @@ static int btrfsic_test_for_metadata(struct btrfsic_state *state, char **datav, unsigned int num_pages) { struct btrfs_fs_info *fs_info = state->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); struct btrfs_header *h; u8 csum[BTRFS_CSUM_SIZE]; - u32 crc = ~(u32)0; unsigned int i; if (num_pages * PAGE_SIZE < state->metablock_size) @@ -1723,14 +1723,17 @@ static int btrfsic_test_for_metadata(struct btrfsic_state *state, if (memcmp(h->fsid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE)) return 1; + shash->tfm = fs_info->csum_shash; + crypto_shash_init(shash); + for (i = 0; i < num_pages; i++) { u8 *data = i ? datav[i] : (datav[i] + BTRFS_CSUM_SIZE); size_t sublen = i ? PAGE_SIZE : (PAGE_SIZE - BTRFS_CSUM_SIZE); - crc = crc32c(crc, data, sublen); + crypto_shash_update(shash, data, sublen); } - btrfs_csum_final(crc, csum); + crypto_shash_final(shash, csum); if (memcmp(csum, h->csum, state->csum_size)) return 1; diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 84dd4a8980c5..60c47b417a4b 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -17,6 +17,7 @@ #include <linux/slab.h> #include <linux/sched/mm.h> #include <linux/log2.h> +#include <crypto/hash.h> #include "ctree.h" #include "disk-io.h" #include "transaction.h" @@ -42,6 +43,22 @@ const char* btrfs_compress_type2str(enum btrfs_compression_type type) return NULL; } +bool btrfs_compress_is_valid_type(const char *str, size_t len) +{ + int i; + + for (i = 1; i < ARRAY_SIZE(btrfs_compress_types); i++) { + size_t comp_len = strlen(btrfs_compress_types[i]); + + if (len < comp_len) + continue; + + if (!strncmp(btrfs_compress_types[i], str, comp_len)) + return true; + } + return false; +} + static int btrfs_decompress_bio(struct compressed_bio *cb); static inline int compressed_bio_size(struct btrfs_fs_info *fs_info, @@ -57,32 +74,37 @@ static int check_compressed_csum(struct btrfs_inode *inode, struct compressed_bio *cb, u64 disk_start) { + struct btrfs_fs_info *fs_info = inode->root->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); + const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); int ret; struct page *page; unsigned long i; char *kaddr; - u32 csum; - u32 *cb_sum = &cb->sums; + u8 csum[BTRFS_CSUM_SIZE]; + u8 *cb_sum = cb->sums; if (inode->flags & BTRFS_INODE_NODATASUM) return 0; + shash->tfm = fs_info->csum_shash; + for (i = 0; i < cb->nr_pages; i++) { page = cb->compressed_pages[i]; - csum = ~(u32)0; + crypto_shash_init(shash); kaddr = kmap_atomic(page); - csum = btrfs_csum_data(kaddr, csum, PAGE_SIZE); - btrfs_csum_final(csum, (u8 *)&csum); + crypto_shash_update(shash, kaddr, PAGE_SIZE); kunmap_atomic(kaddr); + crypto_shash_final(shash, (u8 *)&csum); - if (csum != *cb_sum) { - btrfs_print_data_csum_error(inode, disk_start, csum, - *cb_sum, cb->mirror_num); + if (memcmp(&csum, cb_sum, csum_size)) { + btrfs_print_data_csum_error(inode, disk_start, + csum, cb_sum, cb->mirror_num); ret = -EIO; goto fail; } - cb_sum++; + cb_sum += csum_size; } ret = 0; @@ -318,7 +340,8 @@ blk_status_t btrfs_submit_compressed_write(struct inode *inode, u64 start, bdev = fs_info->fs_devices->latest_bdev; - bio = btrfs_bio_alloc(bdev, first_byte); + bio = btrfs_bio_alloc(first_byte); + bio_set_dev(bio, bdev); bio->bi_opf = REQ_OP_WRITE | write_flags; bio->bi_private = cb; bio->bi_end_io = end_compressed_bio_write; @@ -360,7 +383,8 @@ blk_status_t btrfs_submit_compressed_write(struct inode *inode, u64 start, bio_endio(bio); } - bio = btrfs_bio_alloc(bdev, first_byte); + bio = btrfs_bio_alloc(first_byte); + bio_set_dev(bio, bdev); bio->bi_opf = REQ_OP_WRITE | write_flags; bio->bi_private = cb; bio->bi_end_io = end_compressed_bio_write; @@ -536,7 +560,8 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, struct extent_map *em; blk_status_t ret = BLK_STS_RESOURCE; int faili = 0; - u32 *sums; + const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + u8 *sums; em_tree = &BTRFS_I(inode)->extent_tree; @@ -558,7 +583,7 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, cb->errors = 0; cb->inode = inode; cb->mirror_num = mirror_num; - sums = &cb->sums; + sums = cb->sums; cb->start = em->orig_start; em_len = em->len; @@ -597,7 +622,8 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, /* include any pages we added in add_ra-bio_pages */ cb->len = bio->bi_iter.bi_size; - comp_bio = btrfs_bio_alloc(bdev, cur_disk_byte); + comp_bio = btrfs_bio_alloc(cur_disk_byte); + bio_set_dev(comp_bio, bdev); comp_bio->bi_opf = REQ_OP_READ; comp_bio->bi_private = cb; comp_bio->bi_end_io = end_compressed_bio_read; @@ -617,6 +643,8 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, page->mapping = NULL; if (submit || bio_add_page(comp_bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { + unsigned int nr_sectors; + ret = btrfs_bio_wq_end_io(fs_info, comp_bio, BTRFS_WQ_ENDIO_DATA); BUG_ON(ret); /* -ENOMEM */ @@ -634,8 +662,10 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, sums); BUG_ON(ret); /* -ENOMEM */ } - sums += DIV_ROUND_UP(comp_bio->bi_iter.bi_size, - fs_info->sectorsize); + + nr_sectors = DIV_ROUND_UP(comp_bio->bi_iter.bi_size, + fs_info->sectorsize); + sums += csum_size * nr_sectors; ret = btrfs_map_bio(fs_info, comp_bio, mirror_num, 0); if (ret) { @@ -643,7 +673,8 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, bio_endio(comp_bio); } - comp_bio = btrfs_bio_alloc(bdev, cur_disk_byte); + comp_bio = btrfs_bio_alloc(cur_disk_byte); + bio_set_dev(comp_bio, bdev); comp_bio->bi_opf = REQ_OP_READ; comp_bio->bi_private = cb; comp_bio->bi_end_io = end_compressed_bio_read; diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index 9976fe0f7526..2035b8eb1290 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -61,7 +61,7 @@ struct compressed_bio { * the start of a variable length array of checksums only * used by reads */ - u32 sums; + u8 sums[]; }; static inline unsigned int btrfs_compress_type(unsigned int type_level) @@ -173,6 +173,7 @@ extern const struct btrfs_compress_op btrfs_lzo_compress; extern const struct btrfs_compress_op btrfs_zstd_compress; const char* btrfs_compress_type2str(enum btrfs_compression_type type); +bool btrfs_compress_is_valid_type(const char *str, size_t len); int btrfs_compress_heuristic(struct inode *inode, u64 start, u64 end); diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 0a61dff27f57..299e11e6c554 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -19,6 +19,7 @@ #include <linux/kobject.h> #include <trace/events/btrfs.h> #include <asm/kmap_types.h> +#include <asm/unaligned.h> #include <linux/pagemap.h> #include <linux/btrfs.h> #include <linux/btrfs_tree.h> @@ -31,11 +32,13 @@ #include "extent_io.h" #include "extent_map.h" #include "async-thread.h" +#include "block-rsv.h" struct btrfs_trans_handle; struct btrfs_transaction; struct btrfs_pending_snapshot; struct btrfs_delayed_ref_root; +struct btrfs_space_info; extern struct kmem_cache *btrfs_trans_handle_cachep; extern struct kmem_cache *btrfs_bit_radix_cachep; extern struct kmem_cache *btrfs_path_cachep; @@ -45,7 +48,16 @@ struct btrfs_ref; #define BTRFS_MAGIC 0x4D5F53665248425FULL /* ascii _BHRfS_M, no null */ -#define BTRFS_MAX_MIRRORS 3 +/* + * Maximum number of mirrors that can be available for all profiles counting + * the target device of dev-replace as one. During an active device replace + * procedure, the target device of the copy operation is a mirror for the + * filesystem data as well that can be used to read data in order to repair + * read errors on other disks. + * + * Current value is derived from RAID1 with 2 copies. + */ +#define BTRFS_MAX_MIRRORS (2 + 1) #define BTRFS_MAX_LEVEL 8 @@ -72,6 +84,7 @@ struct btrfs_ref; /* four bytes for CRC32 */ static const int btrfs_csum_sizes[] = { 4 }; +static const char *btrfs_csum_names[] = { "crc32c" }; #define BTRFS_EMPTY_DIR_SIZE 0 @@ -99,10 +112,6 @@ static inline u32 count_max_extents(u64 size) return div_u64(size + BTRFS_MAX_EXTENT_SIZE - 1, BTRFS_MAX_EXTENT_SIZE); } -struct btrfs_mapping_tree { - struct extent_map_tree map_tree; -}; - static inline unsigned long btrfs_chunk_item_size(int num_stripes) { BUG_ON(num_stripes == 0); @@ -395,115 +404,6 @@ struct raid_kobject { struct list_head list; }; -struct btrfs_space_info { - spinlock_t lock; - - u64 total_bytes; /* total bytes in the space, - this doesn't take mirrors into account */ - u64 bytes_used; /* total bytes used, - this doesn't take mirrors into account */ - u64 bytes_pinned; /* total bytes pinned, will be freed when the - transaction finishes */ - u64 bytes_reserved; /* total bytes the allocator has reserved for - current allocations */ - u64 bytes_may_use; /* number of bytes that may be used for - delalloc/allocations */ - u64 bytes_readonly; /* total bytes that are read only */ - - u64 max_extent_size; /* This will hold the maximum extent size of - the space info if we had an ENOSPC in the - allocator. */ - - unsigned int full:1; /* indicates that we cannot allocate any more - chunks for this space */ - unsigned int chunk_alloc:1; /* set if we are allocating a chunk */ - - unsigned int flush:1; /* set if we are trying to make space */ - - unsigned int force_alloc; /* set if we need to force a chunk - alloc for this space */ - - u64 disk_used; /* total bytes used on disk */ - u64 disk_total; /* total bytes on disk, takes mirrors into - account */ - - u64 flags; - - /* - * bytes_pinned is kept in line with what is actually pinned, as in - * we've called update_block_group and dropped the bytes_used counter - * and increased the bytes_pinned counter. However this means that - * bytes_pinned does not reflect the bytes that will be pinned once the - * delayed refs are flushed, so this counter is inc'ed every time we - * call btrfs_free_extent so it is a realtime count of what will be - * freed once the transaction is committed. It will be zeroed every - * time the transaction commits. - */ - struct percpu_counter total_bytes_pinned; - - struct list_head list; - /* Protected by the spinlock 'lock'. */ - struct list_head ro_bgs; - struct list_head priority_tickets; - struct list_head tickets; - /* - * tickets_id just indicates the next ticket will be handled, so note - * it's not stored per ticket. - */ - u64 tickets_id; - - struct rw_semaphore groups_sem; - /* for block groups in our same type */ - struct list_head block_groups[BTRFS_NR_RAID_TYPES]; - wait_queue_head_t wait; - - struct kobject kobj; - struct kobject *block_group_kobjs[BTRFS_NR_RAID_TYPES]; -}; - -/* - * Types of block reserves - */ -enum { - BTRFS_BLOCK_RSV_GLOBAL, - BTRFS_BLOCK_RSV_DELALLOC, - BTRFS_BLOCK_RSV_TRANS, - BTRFS_BLOCK_RSV_CHUNK, - BTRFS_BLOCK_RSV_DELOPS, - BTRFS_BLOCK_RSV_DELREFS, - BTRFS_BLOCK_RSV_EMPTY, - BTRFS_BLOCK_RSV_TEMP, -}; - -struct btrfs_block_rsv { - u64 size; - u64 reserved; - struct btrfs_space_info *space_info; - spinlock_t lock; - unsigned short full; - unsigned short type; - unsigned short failfast; - - /* - * Qgroup equivalent for @size @reserved - * - * Unlike normal @size/@reserved for inode rsv, qgroup doesn't care - * about things like csum size nor how many tree blocks it will need to - * reserve. - * - * Qgroup cares more about net change of the extent usage. - * - * So for one newly inserted file extent, in worst case it will cause - * leaf split and level increase, nodesize for each file extent is - * already too much. - * - * In short, qgroup_size/reserved is the upper limit of possible needed - * qgroup metadata reservation. - */ - u64 qgroup_rsv_size; - u64 qgroup_rsv_reserved; -}; - /* * free clusters are used to claim free space in relatively large chunks, * allowing us to do less seeky writes. They are used for all metadata @@ -786,11 +686,18 @@ enum { /* * Indicate that balance has been set up from the ioctl and is in the * main phase. The fs_info::balance_ctl is initialized. + * Set and cleared while holding fs_info::balance_mutex. */ BTRFS_FS_BALANCE_RUNNING, /* Indicate that the cleaner thread is awake and doing something. */ BTRFS_FS_CLEANER_RUNNING, + + /* + * The checksumming has an optimized version and is considered fast, + * so we don't need to offload checksums to workqueues. + */ + BTRFS_FS_CSUM_IMPL_FAST, }; struct btrfs_fs_info { @@ -824,7 +731,7 @@ struct btrfs_fs_info { struct extent_io_tree *pinned_extents; /* logical->physical extent mapping */ - struct btrfs_mapping_tree mapping_tree; + struct extent_map_tree mapping_tree; /* * block reservation for extent, checksum, root tree and @@ -1160,6 +1067,14 @@ struct btrfs_fs_info { spinlock_t swapfile_pins_lock; struct rb_root swapfile_pins; + struct crypto_shash *csum_shash; + + /* + * Number of send operations in progress. + * Updated while holding fs_info::balance_mutex. + */ + int send_in_progress; + #ifdef CONFIG_BTRFS_FS_REF_VERIFY spinlock_t ref_verify_lock; struct rb_root block_tree; @@ -2451,6 +2366,11 @@ static inline int btrfs_super_csum_size(const struct btrfs_super_block *s) return btrfs_csum_sizes[t]; } +static inline const char *btrfs_super_csum_name(u16 csum_type) +{ + /* csum type is validated at mount time */ + return btrfs_csum_names[csum_type]; +} /* * The leaf data grows from end-to-front in the node. @@ -2642,6 +2562,16 @@ BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_cursor_right, ((unsigned long)(BTRFS_LEAF_DATA_OFFSET + \ btrfs_item_offset_nr(leaf, slot))) +static inline u32 btrfs_crc32c(u32 crc, const void *address, unsigned length) +{ + return crc32c(crc, address, length); +} + +static inline void btrfs_crc32c_final(u32 crc, u8 *result) +{ + put_unaligned_le32(~crc, result); +} + static inline u64 btrfs_name_hash(const char *name, int len) { return crc32c((u32)~1, name, len); @@ -2656,12 +2586,6 @@ static inline u64 btrfs_extref_hash(u64 parent_objectid, const char *name, return (u64) crc32c(parent_objectid, name, len); } -static inline bool btrfs_mixed_space_info(struct btrfs_space_info *space_info) -{ - return ((space_info->flags & BTRFS_BLOCK_GROUP_METADATA) && - (space_info->flags & BTRFS_BLOCK_GROUP_DATA)); -} - static inline gfp_t btrfs_alloc_write_mask(struct address_space *mapping) { return mapping_gfp_constraint(mapping, ~__GFP_FS); @@ -2698,8 +2622,6 @@ static inline u64 btrfs_calc_trunc_metadata_size(struct btrfs_fs_info *fs_info, return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * num_items; } -int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans); -bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info); void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info, const u64 start); void btrfs_wait_block_group_reservations(struct btrfs_block_group_cache *bg); @@ -2814,17 +2736,28 @@ enum btrfs_flush_state { COMMIT_TRANS = 9, }; -int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes); -int btrfs_check_data_free_space(struct inode *inode, - struct extent_changeset **reserved, u64 start, u64 len); -void btrfs_free_reserved_data_space(struct inode *inode, - struct extent_changeset *reserved, u64 start, u64 len); -void btrfs_delalloc_release_space(struct inode *inode, - struct extent_changeset *reserved, - u64 start, u64 len, bool qgroup_free); -void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start, - u64 len); -void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans); +/* + * control flags for do_chunk_alloc's force field + * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk + * if we really need one. + * + * CHUNK_ALLOC_LIMITED means to only try and allocate one + * if we have very few chunks already allocated. This is + * used as part of the clustering code to help make sure + * we have a good pool of storage to cluster in, without + * filling the FS with empty chunks + * + * CHUNK_ALLOC_FORCE means it must try to allocate one + * + */ +enum btrfs_chunk_alloc_enum { + CHUNK_ALLOC_NO_FORCE, + CHUNK_ALLOC_LIMITED, + CHUNK_ALLOC_FORCE, +}; + +int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, + enum btrfs_chunk_alloc_enum force); int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, struct btrfs_block_rsv *rsv, int nitems, bool use_global_rsv); @@ -2834,41 +2767,6 @@ void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes, bool qgroup_free); int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes); -void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes, - bool qgroup_free); -int btrfs_delalloc_reserve_space(struct inode *inode, - struct extent_changeset **reserved, u64 start, u64 len); -void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type); -struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, - unsigned short type); -void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *rsv, - unsigned short type); -void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *rsv); -int btrfs_block_rsv_add(struct btrfs_root *root, - struct btrfs_block_rsv *block_rsv, u64 num_bytes, - enum btrfs_reserve_flush_enum flush); -int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor); -int btrfs_block_rsv_refill(struct btrfs_root *root, - struct btrfs_block_rsv *block_rsv, u64 min_reserved, - enum btrfs_reserve_flush_enum flush); -int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv, - struct btrfs_block_rsv *dst_rsv, u64 num_bytes, - bool update_size); -int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *dest, u64 num_bytes, - int min_factor); -void btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *block_rsv, - u64 num_bytes); -void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr); -void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans); -int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info, - enum btrfs_reserve_flush_enum flush); -void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *src, - u64 num_bytes); int btrfs_inc_block_group_ro(struct btrfs_block_group_cache *cache); void btrfs_dec_block_group_ro(struct btrfs_block_group_cache *cache); void btrfs_put_block_group_cache(struct btrfs_fs_info *info); @@ -3186,7 +3084,8 @@ int btrfs_find_name_in_ext_backref(struct extent_buffer *leaf, int slot, struct btrfs_dio_private; int btrfs_del_csums(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info, u64 bytenr, u64 len); -blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u32 *dst); +blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, + u8 *dst); blk_status_t btrfs_lookup_bio_sums_dio(struct inode *inode, struct bio *bio, u64 logical_offset); int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, @@ -3514,8 +3413,7 @@ __cold static inline void assfail(const char *expr, const char *file, int line) { if (IS_ENABLED(CONFIG_BTRFS_ASSERT)) { - pr_err("assertion failed: %s, file: %s, line: %d\n", - expr, file, line); + pr_err("assertion failed: %s, in %s:%d\n", expr, file, line); BUG(); } } @@ -3599,10 +3497,11 @@ do { \ /* compatibility and incompatibility defines */ #define btrfs_set_fs_incompat(__fs_info, opt) \ - __btrfs_set_fs_incompat((__fs_info), BTRFS_FEATURE_INCOMPAT_##opt) + __btrfs_set_fs_incompat((__fs_info), BTRFS_FEATURE_INCOMPAT_##opt, \ + #opt) static inline void __btrfs_set_fs_incompat(struct btrfs_fs_info *fs_info, - u64 flag) + u64 flag, const char* name) { struct btrfs_super_block *disk_super; u64 features; @@ -3615,18 +3514,20 @@ static inline void __btrfs_set_fs_incompat(struct btrfs_fs_info *fs_info, if (!(features & flag)) { features |= flag; btrfs_set_super_incompat_flags(disk_super, features); - btrfs_info(fs_info, "setting %llu feature flag", - flag); + btrfs_info(fs_info, + "setting incompat feature flag for %s (0x%llx)", + name, flag); } spin_unlock(&fs_info->super_lock); } } #define btrfs_clear_fs_incompat(__fs_info, opt) \ - __btrfs_clear_fs_incompat((__fs_info), BTRFS_FEATURE_INCOMPAT_##opt) + __btrfs_clear_fs_incompat((__fs_info), BTRFS_FEATURE_INCOMPAT_##opt, \ + #opt) static inline void __btrfs_clear_fs_incompat(struct btrfs_fs_info *fs_info, - u64 flag) + u64 flag, const char* name) { struct btrfs_super_block *disk_super; u64 features; @@ -3639,8 +3540,9 @@ static inline void __btrfs_clear_fs_incompat(struct btrfs_fs_info *fs_info, if (features & flag) { features &= ~flag; btrfs_set_super_incompat_flags(disk_super, features); - btrfs_info(fs_info, "clearing %llu feature flag", - flag); + btrfs_info(fs_info, + "clearing incompat feature flag for %s (0x%llx)", + name, flag); } spin_unlock(&fs_info->super_lock); } @@ -3657,10 +3559,11 @@ static inline bool __btrfs_fs_incompat(struct btrfs_fs_info *fs_info, u64 flag) } #define btrfs_set_fs_compat_ro(__fs_info, opt) \ - __btrfs_set_fs_compat_ro((__fs_info), BTRFS_FEATURE_COMPAT_RO_##opt) + __btrfs_set_fs_compat_ro((__fs_info), BTRFS_FEATURE_COMPAT_RO_##opt, \ + #opt) static inline void __btrfs_set_fs_compat_ro(struct btrfs_fs_info *fs_info, - u64 flag) + u64 flag, const char *name) { struct btrfs_super_block *disk_super; u64 features; @@ -3673,18 +3576,20 @@ static inline void __btrfs_set_fs_compat_ro(struct btrfs_fs_info *fs_info, if (!(features & flag)) { features |= flag; btrfs_set_super_compat_ro_flags(disk_super, features); - btrfs_info(fs_info, "setting %llu ro feature flag", - flag); + btrfs_info(fs_info, + "setting compat-ro feature flag for %s (0x%llx)", + name, flag); } spin_unlock(&fs_info->super_lock); } } #define btrfs_clear_fs_compat_ro(__fs_info, opt) \ - __btrfs_clear_fs_compat_ro((__fs_info), BTRFS_FEATURE_COMPAT_RO_##opt) + __btrfs_clear_fs_compat_ro((__fs_info), BTRFS_FEATURE_COMPAT_RO_##opt, \ + #opt) static inline void __btrfs_clear_fs_compat_ro(struct btrfs_fs_info *fs_info, - u64 flag) + u64 flag, const char *name) { struct btrfs_super_block *disk_super; u64 features; @@ -3697,8 +3602,9 @@ static inline void __btrfs_clear_fs_compat_ro(struct btrfs_fs_info *fs_info, if (features & flag) { features &= ~flag; btrfs_set_super_compat_ro_flags(disk_super, features); - btrfs_info(fs_info, "clearing %llu ro feature flag", - flag); + btrfs_info(fs_info, + "clearing compat-ro feature flag for %s (0x%llx)", + name, flag); } spin_unlock(&fs_info->super_lock); } diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c new file mode 100644 index 000000000000..17f7c0d38768 --- /dev/null +++ b/fs/btrfs/delalloc-space.c @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "ctree.h" +#include "delalloc-space.h" +#include "block-rsv.h" +#include "btrfs_inode.h" +#include "space-info.h" +#include "transaction.h" +#include "qgroup.h" + +int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_space_info *data_sinfo = fs_info->data_sinfo; + u64 used; + int ret = 0; + int need_commit = 2; + int have_pinned_space; + + /* Make sure bytes are sectorsize aligned */ + bytes = ALIGN(bytes, fs_info->sectorsize); + + if (btrfs_is_free_space_inode(inode)) { + need_commit = 0; + ASSERT(current->journal_info); + } + +again: + /* Make sure we have enough space to handle the data first */ + spin_lock(&data_sinfo->lock); + used = btrfs_space_info_used(data_sinfo, true); + + if (used + bytes > data_sinfo->total_bytes) { + struct btrfs_trans_handle *trans; + + /* + * If we don't have enough free bytes in this space then we need + * to alloc a new chunk. + */ + if (!data_sinfo->full) { + u64 alloc_target; + + data_sinfo->force_alloc = CHUNK_ALLOC_FORCE; + spin_unlock(&data_sinfo->lock); + + alloc_target = btrfs_data_alloc_profile(fs_info); + /* + * It is ugly that we don't call nolock join + * transaction for the free space inode case here. + * But it is safe because we only do the data space + * reservation for the free space cache in the + * transaction context, the common join transaction + * just increase the counter of the current transaction + * handler, doesn't try to acquire the trans_lock of + * the fs. + */ + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_chunk_alloc(trans, alloc_target, + CHUNK_ALLOC_NO_FORCE); + btrfs_end_transaction(trans); + if (ret < 0) { + if (ret != -ENOSPC) + return ret; + else { + have_pinned_space = 1; + goto commit_trans; + } + } + + goto again; + } + + /* + * If we don't have enough pinned space to deal with this + * allocation, and no removed chunk in current transaction, + * don't bother committing the transaction. + */ + have_pinned_space = __percpu_counter_compare( + &data_sinfo->total_bytes_pinned, + used + bytes - data_sinfo->total_bytes, + BTRFS_TOTAL_BYTES_PINNED_BATCH); + spin_unlock(&data_sinfo->lock); + + /* Commit the current transaction and try again */ +commit_trans: + if (need_commit) { + need_commit--; + + if (need_commit > 0) { + btrfs_start_delalloc_roots(fs_info, -1); + btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, + (u64)-1); + } + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) + return PTR_ERR(trans); + if (have_pinned_space >= 0 || + test_bit(BTRFS_TRANS_HAVE_FREE_BGS, + &trans->transaction->flags) || + need_commit > 0) { + ret = btrfs_commit_transaction(trans); + if (ret) + return ret; + /* + * The cleaner kthread might still be doing iput + * operations. Wait for it to finish so that + * more space is released. We don't need to + * explicitly run the delayed iputs here because + * the commit_transaction would have woken up + * the cleaner. + */ + ret = btrfs_wait_on_delayed_iputs(fs_info); + if (ret) + return ret; + goto again; + } else { + btrfs_end_transaction(trans); + } + } + + trace_btrfs_space_reservation(fs_info, + "space_info:enospc", + data_sinfo->flags, bytes, 1); + return -ENOSPC; + } + btrfs_space_info_update_bytes_may_use(fs_info, data_sinfo, bytes); + trace_btrfs_space_reservation(fs_info, "space_info", + data_sinfo->flags, bytes, 1); + spin_unlock(&data_sinfo->lock); + + return 0; +} + +int btrfs_check_data_free_space(struct inode *inode, + struct extent_changeset **reserved, u64 start, u64 len) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + int ret; + + /* align the range */ + len = round_up(start + len, fs_info->sectorsize) - + round_down(start, fs_info->sectorsize); + start = round_down(start, fs_info->sectorsize); + + ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode), len); + if (ret < 0) + return ret; + + /* Use new btrfs_qgroup_reserve_data to reserve precious data space. */ + ret = btrfs_qgroup_reserve_data(inode, reserved, start, len); + if (ret < 0) + btrfs_free_reserved_data_space_noquota(inode, start, len); + else + ret = 0; + return ret; +} + +/* + * Called if we need to clear a data reservation for this inode + * Normally in a error case. + * + * This one will *NOT* use accurate qgroup reserved space API, just for case + * which we can't sleep and is sure it won't affect qgroup reserved space. + * Like clear_bit_hook(). + */ +void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start, + u64 len) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + struct btrfs_space_info *data_sinfo; + + /* Make sure the range is aligned to sectorsize */ + len = round_up(start + len, fs_info->sectorsize) - + round_down(start, fs_info->sectorsize); + start = round_down(start, fs_info->sectorsize); + + data_sinfo = fs_info->data_sinfo; + spin_lock(&data_sinfo->lock); + btrfs_space_info_update_bytes_may_use(fs_info, data_sinfo, -len); + trace_btrfs_space_reservation(fs_info, "space_info", + data_sinfo->flags, len, 0); + spin_unlock(&data_sinfo->lock); +} + +/* + * Called if we need to clear a data reservation for this inode + * Normally in a error case. + * + * This one will handle the per-inode data rsv map for accurate reserved + * space framework. + */ +void btrfs_free_reserved_data_space(struct inode *inode, + struct extent_changeset *reserved, u64 start, u64 len) +{ + struct btrfs_root *root = BTRFS_I(inode)->root; + + /* Make sure the range is aligned to sectorsize */ + len = round_up(start + len, root->fs_info->sectorsize) - + round_down(start, root->fs_info->sectorsize); + start = round_down(start, root->fs_info->sectorsize); + + btrfs_free_reserved_data_space_noquota(inode, start, len); + btrfs_qgroup_free_data(inode, reserved, start, len); +} + +/** + * btrfs_inode_rsv_release - release any excessive reservation. + * @inode - the inode we need to release from. + * @qgroup_free - free or convert qgroup meta. + * Unlike normal operation, qgroup meta reservation needs to know if we are + * freeing qgroup reservation or just converting it into per-trans. Normally + * @qgroup_free is true for error handling, and false for normal release. + * + * This is the same as btrfs_block_rsv_release, except that it handles the + * tracepoint for the reservation. + */ +static void btrfs_inode_rsv_release(struct btrfs_inode *inode, bool qgroup_free) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_block_rsv *block_rsv = &inode->block_rsv; + u64 released = 0; + u64 qgroup_to_release = 0; + + /* + * Since we statically set the block_rsv->size we just want to say we + * are releasing 0 bytes, and then we'll just get the reservation over + * the size free'd. + */ + released = __btrfs_block_rsv_release(fs_info, block_rsv, 0, + &qgroup_to_release); + if (released > 0) + trace_btrfs_space_reservation(fs_info, "delalloc", + btrfs_ino(inode), released, 0); + if (qgroup_free) + btrfs_qgroup_free_meta_prealloc(inode->root, qgroup_to_release); + else + btrfs_qgroup_convert_reserved_meta(inode->root, + qgroup_to_release); +} + +static void btrfs_calculate_inode_block_rsv_size(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode) +{ + struct btrfs_block_rsv *block_rsv = &inode->block_rsv; + u64 reserve_size = 0; + u64 qgroup_rsv_size = 0; + u64 csum_leaves; + unsigned outstanding_extents; + + lockdep_assert_held(&inode->lock); + outstanding_extents = inode->outstanding_extents; + if (outstanding_extents) + reserve_size = btrfs_calc_trans_metadata_size(fs_info, + outstanding_extents + 1); + csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, + inode->csum_bytes); + reserve_size += btrfs_calc_trans_metadata_size(fs_info, + csum_leaves); + /* + * For qgroup rsv, the calculation is very simple: + * account one nodesize for each outstanding extent + * + * This is overestimating in most cases. + */ + qgroup_rsv_size = (u64)outstanding_extents * fs_info->nodesize; + + spin_lock(&block_rsv->lock); + block_rsv->size = reserve_size; + block_rsv->qgroup_rsv_size = qgroup_rsv_size; + spin_unlock(&block_rsv->lock); +} + +static void calc_inode_reservations(struct btrfs_fs_info *fs_info, + u64 num_bytes, u64 *meta_reserve, + u64 *qgroup_reserve) +{ + u64 nr_extents = count_max_extents(num_bytes); + u64 csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, num_bytes); + + /* We add one for the inode update at finish ordered time */ + *meta_reserve = btrfs_calc_trans_metadata_size(fs_info, + nr_extents + csum_leaves + 1); + *qgroup_reserve = nr_extents * fs_info->nodesize; +} + +int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *block_rsv = &inode->block_rsv; + u64 meta_reserve, qgroup_reserve; + unsigned nr_extents; + enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL; + int ret = 0; + bool delalloc_lock = true; + + /* + * If we are a free space inode we need to not flush since we will be in + * the middle of a transaction commit. We also don't need the delalloc + * mutex since we won't race with anybody. We need this mostly to make + * lockdep shut its filthy mouth. + * + * If we have a transaction open (can happen if we call truncate_block + * from truncate), then we need FLUSH_LIMIT so we don't deadlock. + */ + if (btrfs_is_free_space_inode(inode)) { + flush = BTRFS_RESERVE_NO_FLUSH; + delalloc_lock = false; + } else { + if (current->journal_info) + flush = BTRFS_RESERVE_FLUSH_LIMIT; + + if (btrfs_transaction_in_commit(fs_info)) + schedule_timeout(1); + } + + if (delalloc_lock) + mutex_lock(&inode->delalloc_mutex); + + num_bytes = ALIGN(num_bytes, fs_info->sectorsize); + + /* + * We always want to do it this way, every other way is wrong and ends + * in tears. Pre-reserving the amount we are going to add will always + * be the right way, because otherwise if we have enough parallelism we + * could end up with thousands of inodes all holding little bits of + * reservations they were able to make previously and the only way to + * reclaim that space is to ENOSPC out the operations and clear + * everything out and try again, which is bad. This way we just + * over-reserve slightly, and clean up the mess when we are done. + */ + calc_inode_reservations(fs_info, num_bytes, &meta_reserve, + &qgroup_reserve); + ret = btrfs_qgroup_reserve_meta_prealloc(root, qgroup_reserve, true); + if (ret) + goto out_fail; + ret = btrfs_reserve_metadata_bytes(root, block_rsv, meta_reserve, flush); + if (ret) + goto out_qgroup; + + /* + * Now we need to update our outstanding extents and csum bytes _first_ + * and then add the reservation to the block_rsv. This keeps us from + * racing with an ordered completion or some such that would think it + * needs to free the reservation we just made. + */ + spin_lock(&inode->lock); + nr_extents = count_max_extents(num_bytes); + btrfs_mod_outstanding_extents(inode, nr_extents); + inode->csum_bytes += num_bytes; + btrfs_calculate_inode_block_rsv_size(fs_info, inode); + spin_unlock(&inode->lock); + + /* Now we can safely add our space to our block rsv */ + btrfs_block_rsv_add_bytes(block_rsv, meta_reserve, false); + trace_btrfs_space_reservation(root->fs_info, "delalloc", + btrfs_ino(inode), meta_reserve, 1); + + spin_lock(&block_rsv->lock); + block_rsv->qgroup_rsv_reserved += qgroup_reserve; + spin_unlock(&block_rsv->lock); + + if (delalloc_lock) + mutex_unlock(&inode->delalloc_mutex); + return 0; +out_qgroup: + btrfs_qgroup_free_meta_prealloc(root, qgroup_reserve); +out_fail: + btrfs_inode_rsv_release(inode, true); + if (delalloc_lock) + mutex_unlock(&inode->delalloc_mutex); + return ret; +} + +/** + * btrfs_delalloc_release_metadata - release a metadata reservation for an inode + * @inode: the inode to release the reservation for. + * @num_bytes: the number of bytes we are releasing. + * @qgroup_free: free qgroup reservation or convert it to per-trans reservation + * + * This will release the metadata reservation for an inode. This can be called + * once we complete IO for a given set of bytes to release their metadata + * reservations, or on error for the same reason. + */ +void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes, + bool qgroup_free) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + num_bytes = ALIGN(num_bytes, fs_info->sectorsize); + spin_lock(&inode->lock); + inode->csum_bytes -= num_bytes; + btrfs_calculate_inode_block_rsv_size(fs_info, inode); + spin_unlock(&inode->lock); + + if (btrfs_is_testing(fs_info)) + return; + + btrfs_inode_rsv_release(inode, qgroup_free); +} + +/** + * btrfs_delalloc_release_extents - release our outstanding_extents + * @inode: the inode to balance the reservation for. + * @num_bytes: the number of bytes we originally reserved with + * @qgroup_free: do we need to free qgroup meta reservation or convert them. + * + * When we reserve space we increase outstanding_extents for the extents we may + * add. Once we've set the range as delalloc or created our ordered extents we + * have outstanding_extents to track the real usage, so we use this to free our + * temporarily tracked outstanding_extents. This _must_ be used in conjunction + * with btrfs_delalloc_reserve_metadata. + */ +void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes, + bool qgroup_free) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + unsigned num_extents; + + spin_lock(&inode->lock); + num_extents = count_max_extents(num_bytes); + btrfs_mod_outstanding_extents(inode, -num_extents); + btrfs_calculate_inode_block_rsv_size(fs_info, inode); + spin_unlock(&inode->lock); + + if (btrfs_is_testing(fs_info)) + return; + + btrfs_inode_rsv_release(inode, qgroup_free); +} + +/** + * btrfs_delalloc_reserve_space - reserve data and metadata space for + * delalloc + * @inode: inode we're writing to + * @start: start range we are writing to + * @len: how long the range we are writing to + * @reserved: mandatory parameter, record actually reserved qgroup ranges of + * current reservation. + * + * This will do the following things + * + * - reserve space in data space info for num bytes + * and reserve precious corresponding qgroup space + * (Done in check_data_free_space) + * + * - reserve space for metadata space, based on the number of outstanding + * extents and how much csums will be needed + * also reserve metadata space in a per root over-reserve method. + * - add to the inodes->delalloc_bytes + * - add it to the fs_info's delalloc inodes list. + * (Above 3 all done in delalloc_reserve_metadata) + * + * Return 0 for success + * Return <0 for error(-ENOSPC or -EQUOT) + */ +int btrfs_delalloc_reserve_space(struct inode *inode, + struct extent_changeset **reserved, u64 start, u64 len) +{ + int ret; + + ret = btrfs_check_data_free_space(inode, reserved, start, len); + if (ret < 0) + return ret; + ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len); + if (ret < 0) + btrfs_free_reserved_data_space(inode, *reserved, start, len); + return ret; +} + +/** + * btrfs_delalloc_release_space - release data and metadata space for delalloc + * @inode: inode we're releasing space for + * @start: start position of the space already reserved + * @len: the len of the space already reserved + * @release_bytes: the len of the space we consumed or didn't use + * + * This function will release the metadata space that was not used and will + * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes + * list if there are no delalloc bytes left. + * Also it will handle the qgroup reserved space. + */ +void btrfs_delalloc_release_space(struct inode *inode, + struct extent_changeset *reserved, + u64 start, u64 len, bool qgroup_free) +{ + btrfs_delalloc_release_metadata(BTRFS_I(inode), len, qgroup_free); + btrfs_free_reserved_data_space(inode, reserved, start, len); +} diff --git a/fs/btrfs/delalloc-space.h b/fs/btrfs/delalloc-space.h new file mode 100644 index 000000000000..54466fbd7075 --- /dev/null +++ b/fs/btrfs/delalloc-space.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_DELALLOC_SPACE_H +#define BTRFS_DELALLOC_SPACE_H + +struct extent_changeset; + +int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes); +int btrfs_check_data_free_space(struct inode *inode, + struct extent_changeset **reserved, u64 start, u64 len); +void btrfs_free_reserved_data_space(struct inode *inode, + struct extent_changeset *reserved, u64 start, u64 len); +void btrfs_delalloc_release_space(struct inode *inode, + struct extent_changeset *reserved, + u64 start, u64 len, bool qgroup_free); +void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start, + u64 len); +void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes, + bool qgroup_free); +int btrfs_delalloc_reserve_space(struct inode *inode, + struct extent_changeset **reserved, u64 start, u64 len); + +#endif /* BTRFS_DELALLOC_SPACE_H */ diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index a73fc23e2961..9a91d1eb0af4 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -10,6 +10,7 @@ #include "delayed-ref.h" #include "transaction.h" #include "qgroup.h" +#include "space-info.h" struct kmem_cache *btrfs_delayed_ref_head_cachep; struct kmem_cache *btrfs_delayed_tree_ref_cachep; @@ -24,6 +25,179 @@ struct kmem_cache *btrfs_delayed_extent_op_cachep; * of hammering updates on the extent allocation tree. */ +bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + bool ret = false; + u64 reserved; + + spin_lock(&global_rsv->lock); + reserved = global_rsv->reserved; + spin_unlock(&global_rsv->lock); + + /* + * Since the global reserve is just kind of magic we don't really want + * to rely on it to save our bacon, so if our size is more than the + * delayed_refs_rsv and the global rsv then it's time to think about + * bailing. + */ + spin_lock(&delayed_refs_rsv->lock); + reserved += delayed_refs_rsv->reserved; + if (delayed_refs_rsv->size >= reserved) + ret = true; + spin_unlock(&delayed_refs_rsv->lock); + return ret; +} + +int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans) +{ + u64 num_entries = + atomic_read(&trans->transaction->delayed_refs.num_entries); + u64 avg_runtime; + u64 val; + + smp_mb(); + avg_runtime = trans->fs_info->avg_delayed_ref_runtime; + val = num_entries * avg_runtime; + if (val >= NSEC_PER_SEC) + return 1; + if (val >= NSEC_PER_SEC / 2) + return 2; + + return btrfs_check_space_for_delayed_refs(trans->fs_info); +} + +/** + * btrfs_delayed_refs_rsv_release - release a ref head's reservation. + * @fs_info - the fs_info for our fs. + * @nr - the number of items to drop. + * + * This drops the delayed ref head's count from the delayed refs rsv and frees + * any excess reservation we had. + */ +void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr) +{ + struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv; + u64 num_bytes = btrfs_calc_trans_metadata_size(fs_info, nr); + u64 released = 0; + + released = __btrfs_block_rsv_release(fs_info, block_rsv, num_bytes, + NULL); + if (released) + trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", + 0, released, 0); +} + +/* + * btrfs_update_delayed_refs_rsv - adjust the size of the delayed refs rsv + * @trans - the trans that may have generated delayed refs + * + * This is to be called anytime we may have adjusted trans->delayed_ref_updates, + * it'll calculate the additional size and add it to the delayed_refs_rsv. + */ +void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + u64 num_bytes; + + if (!trans->delayed_ref_updates) + return; + + num_bytes = btrfs_calc_trans_metadata_size(fs_info, + trans->delayed_ref_updates); + spin_lock(&delayed_rsv->lock); + delayed_rsv->size += num_bytes; + delayed_rsv->full = 0; + spin_unlock(&delayed_rsv->lock); + trans->delayed_ref_updates = 0; +} + +/** + * btrfs_migrate_to_delayed_refs_rsv - transfer bytes to our delayed refs rsv. + * @fs_info - the fs info for our fs. + * @src - the source block rsv to transfer from. + * @num_bytes - the number of bytes to transfer. + * + * This transfers up to the num_bytes amount from the src rsv to the + * delayed_refs_rsv. Any extra bytes are returned to the space info. + */ +void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *src, + u64 num_bytes) +{ + struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; + u64 to_free = 0; + + spin_lock(&src->lock); + src->reserved -= num_bytes; + src->size -= num_bytes; + spin_unlock(&src->lock); + + spin_lock(&delayed_refs_rsv->lock); + if (delayed_refs_rsv->size > delayed_refs_rsv->reserved) { + u64 delta = delayed_refs_rsv->size - + delayed_refs_rsv->reserved; + if (num_bytes > delta) { + to_free = num_bytes - delta; + num_bytes = delta; + } + } else { + to_free = num_bytes; + num_bytes = 0; + } + + if (num_bytes) + delayed_refs_rsv->reserved += num_bytes; + if (delayed_refs_rsv->reserved >= delayed_refs_rsv->size) + delayed_refs_rsv->full = 1; + spin_unlock(&delayed_refs_rsv->lock); + + if (num_bytes) + trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", + 0, num_bytes, 1); + if (to_free) + btrfs_space_info_add_old_bytes(fs_info, + delayed_refs_rsv->space_info, to_free); +} + +/** + * btrfs_delayed_refs_rsv_refill - refill based on our delayed refs usage. + * @fs_info - the fs_info for our fs. + * @flush - control how we can flush for this reservation. + * + * This will refill the delayed block_rsv up to 1 items size worth of space and + * will return -ENOSPC if we can't make the reservation. + */ +int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info, + enum btrfs_reserve_flush_enum flush) +{ + struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv; + u64 limit = btrfs_calc_trans_metadata_size(fs_info, 1); + u64 num_bytes = 0; + int ret = -ENOSPC; + + spin_lock(&block_rsv->lock); + if (block_rsv->reserved < block_rsv->size) { + num_bytes = block_rsv->size - block_rsv->reserved; + num_bytes = min(num_bytes, limit); + } + spin_unlock(&block_rsv->lock); + + if (!num_bytes) + return 0; + + ret = btrfs_reserve_metadata_bytes(fs_info->extent_root, block_rsv, + num_bytes, flush); + if (ret) + return ret; + btrfs_block_rsv_add_bytes(block_rsv, num_bytes, 0); + trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", + 0, num_bytes, 1); + return 0; +} + /* * compare two delayed tree backrefs with same bytenr and type */ @@ -957,13 +1131,14 @@ int btrfs_add_delayed_extent_op(struct btrfs_trans_handle *trans, } /* - * this does a simple search for the head node for a given extent. - * It must be called with the delayed ref spinlock held, and it returns - * the head node if any where found, or NULL if not. + * This does a simple search for the head node for a given extent. Returns the + * head node if found, or NULL if not. */ struct btrfs_delayed_ref_head * btrfs_find_delayed_ref_head(struct btrfs_delayed_ref_root *delayed_refs, u64 bytenr) { + lockdep_assert_held(&delayed_refs->lock); + return find_ref_head(delayed_refs, bytenr, false); } diff --git a/fs/btrfs/delayed-ref.h b/fs/btrfs/delayed-ref.h index c18f93ea88ed..1c977e6d45dc 100644 --- a/fs/btrfs/delayed-ref.h +++ b/fs/btrfs/delayed-ref.h @@ -364,6 +364,16 @@ struct btrfs_delayed_ref_head *btrfs_select_ref_head( int btrfs_check_delayed_seq(struct btrfs_fs_info *fs_info, u64 seq); +void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr); +void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans); +int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info, + enum btrfs_reserve_flush_enum flush); +void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *src, + u64 num_bytes); +int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans); +bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info); + /* * helper functions to cast a node into its container */ diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index ee0989c7e3a9..6b2e9aa83ffa 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -201,7 +201,7 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, return PTR_ERR(bdev); } - filemap_write_and_wait(bdev->bd_inode->i_mapping); + sync_blockdev(bdev); devices = &fs_info->fs_devices->devices; list_for_each_entry(device, devices, dev_list) { @@ -237,7 +237,6 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, } rcu_assign_pointer(device->name, name); - mutex_lock(&fs_info->fs_devices->device_list_mutex); set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); device->generation = 0; device->io_width = fs_info->sectorsize; @@ -256,6 +255,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, device->dev_stats_valid = 1; set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE); device->fs_devices = fs_info->fs_devices; + + mutex_lock(&fs_info->fs_devices->device_list_mutex); list_add(&device->dev_list, &fs_info->fs_devices->devices); fs_info->fs_devices->num_devices++; fs_info->fs_devices->open_devices++; @@ -399,7 +400,6 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, int ret; struct btrfs_device *tgt_device = NULL; struct btrfs_device *src_device = NULL; - bool need_unlock; src_device = btrfs_find_device_by_devspec(fs_info, srcdevid, srcdev_name); @@ -413,11 +413,6 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, return -ETXTBSY; } - ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name, - src_device, &tgt_device); - if (ret) - return ret; - /* * Here we commit the transaction to make sure commit_total_bytes * of all the devices are updated. @@ -431,7 +426,11 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, return PTR_ERR(trans); } - need_unlock = true; + ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name, + src_device, &tgt_device); + if (ret) + return ret; + down_write(&dev_replace->rwsem); switch (dev_replace->replace_state) { case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: @@ -442,11 +441,11 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: ASSERT(0); ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED; + up_write(&dev_replace->rwsem); goto leave; } dev_replace->cont_reading_from_srcdev_mode = read_src; - WARN_ON(!src_device); dev_replace->srcdev = src_device; dev_replace->tgtdev = tgt_device; @@ -471,7 +470,6 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, atomic64_set(&dev_replace->num_write_errors, 0); atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0); up_write(&dev_replace->rwsem); - need_unlock = false; ret = btrfs_sysfs_add_device_link(tgt_device->fs_devices, tgt_device); if (ret) @@ -479,16 +477,16 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1); - /* force writing the updated state information to disk */ - trans = btrfs_start_transaction(root, 0); + /* Commit dev_replace state and reserve 1 item for it. */ + trans = btrfs_start_transaction(root, 1); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - need_unlock = true; down_write(&dev_replace->rwsem); dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED; dev_replace->srcdev = NULL; dev_replace->tgtdev = NULL; + up_write(&dev_replace->rwsem); goto leave; } @@ -510,8 +508,6 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, return ret; leave: - if (need_unlock) - up_write(&dev_replace->rwsem); btrfs_destroy_dev_replace_tgtdev(tgt_device); return ret; } @@ -678,7 +674,6 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info, btrfs_device_set_disk_total_bytes(tgt_device, src_device->disk_total_bytes); btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used); - tgt_device->commit_total_bytes = src_device->commit_total_bytes; tgt_device->commit_bytes_used = src_device->bytes_used; btrfs_assign_next_active_device(src_device, tgt_device); @@ -728,7 +723,7 @@ static void btrfs_dev_replace_update_device_in_mapping_tree( struct btrfs_device *srcdev, struct btrfs_device *tgtdev) { - struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree; + struct extent_map_tree *em_tree = &fs_info->mapping_tree; struct extent_map *em; struct map_lookup *map; u64 start = 0; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index deb74a8c191a..41a2bd2e0c56 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -19,6 +19,7 @@ #include <linux/crc32c.h> #include <linux/sched/mm.h> #include <asm/unaligned.h> +#include <crypto/hash.h> #include "ctree.h" #include "disk-io.h" #include "transaction.h" @@ -40,10 +41,6 @@ #include "tree-checker.h" #include "ref-verify.h" -#ifdef CONFIG_X86 -#include <asm/cpufeature.h> -#endif - #define BTRFS_SUPER_FLAG_SUPP (BTRFS_HEADER_FLAG_WRITTEN |\ BTRFS_HEADER_FLAG_RELOC |\ BTRFS_SUPER_FLAG_ERROR |\ @@ -249,16 +246,6 @@ out: return em; } -u32 btrfs_csum_data(const char *data, u32 seed, size_t len) -{ - return crc32c(seed, data, len); -} - -void btrfs_csum_final(u32 crc, u8 *result) -{ - put_unaligned_le32(~crc, result); -} - /* * Compute the csum of a btree block and store the result to provided buffer. * @@ -266,6 +253,8 @@ void btrfs_csum_final(u32 crc, u8 *result) */ static int csum_tree_block(struct extent_buffer *buf, u8 *result) { + struct btrfs_fs_info *fs_info = buf->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); unsigned long len; unsigned long cur_len; unsigned long offset = BTRFS_CSUM_SIZE; @@ -273,9 +262,12 @@ static int csum_tree_block(struct extent_buffer *buf, u8 *result) unsigned long map_start; unsigned long map_len; int err; - u32 crc = ~(u32)0; + + shash->tfm = fs_info->csum_shash; + crypto_shash_init(shash); len = buf->len - offset; + while (len > 0) { /* * Note: we don't need to check for the err == 1 case here, as @@ -288,14 +280,13 @@ static int csum_tree_block(struct extent_buffer *buf, u8 *result) if (WARN_ON(err)) return err; cur_len = min(len, map_len - (offset - map_start)); - crc = btrfs_csum_data(kaddr + offset - map_start, - crc, cur_len); + crypto_shash_update(shash, kaddr + offset - map_start, cur_len); len -= cur_len; offset += cur_len; } memset(result, 0, BTRFS_CSUM_SIZE); - btrfs_csum_final(crc, result); + crypto_shash_final(shash, result); return 0; } @@ -356,6 +347,16 @@ out: return ret; } +static bool btrfs_supported_super_csum(u16 csum_type) +{ + switch (csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + return true; + default: + return false; + } +} + /* * Return 0 if the superblock checksum type matches the checksum value of that * algorithm. Pass the raw disk superblock data. @@ -365,33 +366,25 @@ static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, { struct btrfs_super_block *disk_sb = (struct btrfs_super_block *)raw_disk_sb; - u16 csum_type = btrfs_super_csum_type(disk_sb); - int ret = 0; + char result[BTRFS_CSUM_SIZE]; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); - if (csum_type == BTRFS_CSUM_TYPE_CRC32) { - u32 crc = ~(u32)0; - char result[sizeof(crc)]; + shash->tfm = fs_info->csum_shash; + crypto_shash_init(shash); - /* - * The super_block structure does not span the whole - * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space - * is filled with zeros and is included in the checksum. - */ - crc = btrfs_csum_data(raw_disk_sb + BTRFS_CSUM_SIZE, - crc, BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE); - btrfs_csum_final(crc, result); + /* + * The super_block structure does not span the whole + * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is + * filled with zeros and is included in the checksum. + */ + crypto_shash_update(shash, raw_disk_sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE); + crypto_shash_final(shash, result); - if (memcmp(raw_disk_sb, result, sizeof(result))) - ret = 1; - } + if (memcmp(disk_sb->csum, result, btrfs_super_csum_size(disk_sb))) + return 1; - if (csum_type >= ARRAY_SIZE(btrfs_csum_sizes)) { - btrfs_err(fs_info, "unsupported checksum algorithm %u", - csum_type); - ret = 1; - } - - return ret; + return 0; } int btrfs_verify_level_key(struct extent_buffer *eb, int level, @@ -873,14 +866,13 @@ static blk_status_t btree_submit_bio_start(void *private_data, struct bio *bio, return btree_csum_one_bio(bio); } -static int check_async_write(struct btrfs_inode *bi) +static int check_async_write(struct btrfs_fs_info *fs_info, + struct btrfs_inode *bi) { if (atomic_read(&bi->sync_writers)) return 0; -#ifdef CONFIG_X86 - if (static_cpu_has(X86_FEATURE_XMM4_2)) + if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) return 0; -#endif return 1; } @@ -889,7 +881,7 @@ static blk_status_t btree_submit_bio_hook(struct inode *inode, struct bio *bio, unsigned long bio_flags) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - int async = check_async_write(BTRFS_I(inode)); + int async = check_async_write(fs_info, BTRFS_I(inode)); blk_status_t ret; if (bio_op(bio) != REQ_OP_WRITE) { @@ -2262,6 +2254,29 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info, return 0; } +static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type) +{ + struct crypto_shash *csum_shash; + const char *csum_name = btrfs_super_csum_name(csum_type); + + csum_shash = crypto_alloc_shash(csum_name, 0, 0); + + if (IS_ERR(csum_shash)) { + btrfs_err(fs_info, "error allocating %s hash for checksum", + csum_name); + return PTR_ERR(csum_shash); + } + + fs_info->csum_shash = csum_shash; + + return 0; +} + +static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info) +{ + crypto_free_shash(fs_info->csum_shash); +} + static int btrfs_replay_log(struct btrfs_fs_info *fs_info, struct btrfs_fs_devices *fs_devices) { @@ -2577,7 +2592,7 @@ static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info, ret = validate_super(fs_info, sb, -1); if (ret < 0) goto out; - if (btrfs_super_csum_type(sb) != BTRFS_CSUM_TYPE_CRC32) { + if (!btrfs_supported_super_csum(btrfs_super_csum_type(sb))) { ret = -EUCLEAN; btrfs_err(fs_info, "invalid csum type, has %u want %u", btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32); @@ -2607,6 +2622,7 @@ int open_ctree(struct super_block *sb, u32 stripesize; u64 generation; u64 features; + u16 csum_type; struct btrfs_key location; struct buffer_head *bh; struct btrfs_super_block *disk_super; @@ -2689,7 +2705,7 @@ int open_ctree(struct super_block *sb, INIT_LIST_HEAD(&fs_info->space_info); INIT_LIST_HEAD(&fs_info->tree_mod_seq_list); INIT_LIST_HEAD(&fs_info->unused_bgs); - btrfs_mapping_init(&fs_info->mapping_tree); + extent_map_tree_init(&fs_info->mapping_tree); btrfs_init_block_rsv(&fs_info->global_block_rsv, BTRFS_BLOCK_RSV_GLOBAL); btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS); @@ -2793,6 +2809,8 @@ int open_ctree(struct super_block *sb, spin_lock_init(&fs_info->swapfile_pins_lock); fs_info->swapfile_pins = RB_ROOT; + fs_info->send_in_progress = 0; + ret = btrfs_alloc_stripe_hash_table(fs_info); if (ret) { err = ret; @@ -2813,6 +2831,25 @@ int open_ctree(struct super_block *sb, } /* + * Verify the type first, if that or the the checksum value are + * corrupted, we'll find out + */ + csum_type = btrfs_super_csum_type((struct btrfs_super_block *)bh->b_data); + if (!btrfs_supported_super_csum(csum_type)) { + btrfs_err(fs_info, "unsupported checksum algorithm: %u", + csum_type); + err = -EINVAL; + brelse(bh); + goto fail_alloc; + } + + ret = btrfs_init_csum_hash(fs_info, csum_type); + if (ret) { + err = ret; + goto fail_alloc; + } + + /* * We want to check superblock checksum, the type is stored inside. * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k). */ @@ -2820,7 +2857,7 @@ int open_ctree(struct super_block *sb, btrfs_err(fs_info, "superblock checksum mismatch"); err = -EINVAL; brelse(bh); - goto fail_alloc; + goto fail_csum; } /* @@ -2857,11 +2894,11 @@ int open_ctree(struct super_block *sb, if (ret) { btrfs_err(fs_info, "superblock contains fatal errors"); err = -EINVAL; - goto fail_alloc; + goto fail_csum; } if (!btrfs_super_root(disk_super)) - goto fail_alloc; + goto fail_csum; /* check FS state, whether FS is broken. */ if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR) @@ -2883,7 +2920,7 @@ int open_ctree(struct super_block *sb, ret = btrfs_parse_options(fs_info, options, sb->s_flags); if (ret) { err = ret; - goto fail_alloc; + goto fail_csum; } features = btrfs_super_incompat_flags(disk_super) & @@ -2893,7 +2930,7 @@ int open_ctree(struct super_block *sb, "cannot mount because of unsupported optional features (%llx)", features); err = -EINVAL; - goto fail_alloc; + goto fail_csum; } features = btrfs_super_incompat_flags(disk_super); @@ -2937,7 +2974,7 @@ int open_ctree(struct super_block *sb, btrfs_err(fs_info, "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups", nodesize, sectorsize); - goto fail_alloc; + goto fail_csum; } /* @@ -2953,7 +2990,7 @@ int open_ctree(struct super_block *sb, "cannot mount read-write because of unsupported optional features (%llx)", features); err = -EINVAL; - goto fail_alloc; + goto fail_csum; } ret = btrfs_init_workqueues(fs_info, fs_devices); @@ -3331,6 +3368,8 @@ fail_tree_roots: fail_sb_buffer: btrfs_stop_all_workers(fs_info); btrfs_free_block_groups(fs_info); +fail_csum: + btrfs_free_csum_hash(fs_info); fail_alloc: fail_iput: btrfs_mapping_tree_free(&fs_info->mapping_tree); @@ -3472,17 +3511,20 @@ struct buffer_head *btrfs_read_dev_super(struct block_device *bdev) static int write_dev_supers(struct btrfs_device *device, struct btrfs_super_block *sb, int max_mirrors) { + struct btrfs_fs_info *fs_info = device->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); struct buffer_head *bh; int i; int ret; int errors = 0; - u32 crc; u64 bytenr; int op_flags; if (max_mirrors == 0) max_mirrors = BTRFS_SUPER_MIRROR_MAX; + shash->tfm = fs_info->csum_shash; + for (i = 0; i < max_mirrors; i++) { bytenr = btrfs_sb_offset(i); if (bytenr + BTRFS_SUPER_INFO_SIZE >= @@ -3491,10 +3533,10 @@ static int write_dev_supers(struct btrfs_device *device, btrfs_set_super_bytenr(sb, bytenr); - crc = ~(u32)0; - crc = btrfs_csum_data((const char *)sb + BTRFS_CSUM_SIZE, crc, - BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE); - btrfs_csum_final(crc, sb->csum); + crypto_shash_init(shash); + crypto_shash_update(shash, (const char *)sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE); + crypto_shash_final(shash, sb->csum); /* One reference for us, and we leave it for the caller */ bh = __getblk(device->bdev, bytenr / BTRFS_BDEV_BLOCKSIZE, @@ -3709,7 +3751,7 @@ int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags) if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 || (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE)) - min_tolerated = min(min_tolerated, + min_tolerated = min_t(int, min_tolerated, btrfs_raid_array[BTRFS_RAID_SINGLE]. tolerated_failures); @@ -3718,7 +3760,7 @@ int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags) continue; if (!(flags & btrfs_raid_array[raid_type].bg_flag)) continue; - min_tolerated = min(min_tolerated, + min_tolerated = min_t(int, min_tolerated, btrfs_raid_array[raid_type]. tolerated_failures); } diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index a0161aa1ea0b..e80f7c45a307 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -115,8 +115,6 @@ int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid, int atomic); int btrfs_read_buffer(struct extent_buffer *buf, u64 parent_transid, int level, struct btrfs_key *first_key); -u32 btrfs_csum_data(const char *data, u32 seed, size_t len); -void btrfs_csum_final(u32 crc, u8 *result); blk_status_t btrfs_bio_wq_end_io(struct btrfs_fs_info *info, struct bio *bio, enum btrfs_wq_endio_type metadata); blk_status_t btrfs_wq_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 5faf057f6f37..d3b58e388535 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -28,46 +28,12 @@ #include "sysfs.h" #include "qgroup.h" #include "ref-verify.h" +#include "space-info.h" +#include "block-rsv.h" +#include "delalloc-space.h" #undef SCRAMBLE_DELAYED_REFS -/* - * control flags for do_chunk_alloc's force field - * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk - * if we really need one. - * - * CHUNK_ALLOC_LIMITED means to only try and allocate one - * if we have very few chunks already allocated. This is - * used as part of the clustering code to help make sure - * we have a good pool of storage to cluster in, without - * filling the FS with empty chunks - * - * CHUNK_ALLOC_FORCE means it must try to allocate one - * - */ -enum { - CHUNK_ALLOC_NO_FORCE = 0, - CHUNK_ALLOC_LIMITED = 1, - CHUNK_ALLOC_FORCE = 2, -}; - -/* - * Declare a helper function to detect underflow of various space info members - */ -#define DECLARE_SPACE_INFO_UPDATE(name) \ -static inline void update_##name(struct btrfs_space_info *sinfo, \ - s64 bytes) \ -{ \ - if (bytes < 0 && sinfo->name < -bytes) { \ - WARN_ON(1); \ - sinfo->name = 0; \ - return; \ - } \ - sinfo->name += bytes; \ -} - -DECLARE_SPACE_INFO_UPDATE(bytes_may_use); -DECLARE_SPACE_INFO_UPDATE(bytes_pinned); static int __btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_node *node, u64 parent, @@ -84,21 +50,8 @@ static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_node *node, struct btrfs_delayed_extent_op *extent_op); -static int do_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, - int force); static int find_next_key(struct btrfs_path *path, int level, struct btrfs_key *key); -static void dump_space_info(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *info, u64 bytes, - int dump_block_groups); -static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, - u64 num_bytes); -static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 num_bytes); -static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 num_bytes); static noinline int block_group_cache_done(struct btrfs_block_group_cache *cache) @@ -737,62 +690,39 @@ struct btrfs_block_group_cache *btrfs_lookup_block_group( return block_group_cache_tree_search(info, bytenr, 1); } -static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info, - u64 flags) +static u64 generic_ref_to_space_flags(struct btrfs_ref *ref) { - struct list_head *head = &info->space_info; - struct btrfs_space_info *found; - - flags &= BTRFS_BLOCK_GROUP_TYPE_MASK; - - rcu_read_lock(); - list_for_each_entry_rcu(found, head, list) { - if (found->flags & flags) { - rcu_read_unlock(); - return found; - } + if (ref->type == BTRFS_REF_METADATA) { + if (ref->tree_ref.root == BTRFS_CHUNK_TREE_OBJECTID) + return BTRFS_BLOCK_GROUP_SYSTEM; + else + return BTRFS_BLOCK_GROUP_METADATA; } - rcu_read_unlock(); - return NULL; + return BTRFS_BLOCK_GROUP_DATA; } static void add_pinned_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_ref *ref, int sign) + struct btrfs_ref *ref) { struct btrfs_space_info *space_info; - s64 num_bytes; - u64 flags; - - ASSERT(sign == 1 || sign == -1); - num_bytes = sign * ref->len; - if (ref->type == BTRFS_REF_METADATA) { - if (ref->tree_ref.root == BTRFS_CHUNK_TREE_OBJECTID) - flags = BTRFS_BLOCK_GROUP_SYSTEM; - else - flags = BTRFS_BLOCK_GROUP_METADATA; - } else { - flags = BTRFS_BLOCK_GROUP_DATA; - } + u64 flags = generic_ref_to_space_flags(ref); - space_info = __find_space_info(fs_info, flags); + space_info = btrfs_find_space_info(fs_info, flags); ASSERT(space_info); - percpu_counter_add_batch(&space_info->total_bytes_pinned, num_bytes, + percpu_counter_add_batch(&space_info->total_bytes_pinned, ref->len, BTRFS_TOTAL_BYTES_PINNED_BATCH); } -/* - * after adding space to the filesystem, we need to clear the full flags - * on all the space infos. - */ -void btrfs_clear_space_info_full(struct btrfs_fs_info *info) +static void sub_pinned_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_ref *ref) { - struct list_head *head = &info->space_info; - struct btrfs_space_info *found; + struct btrfs_space_info *space_info; + u64 flags = generic_ref_to_space_flags(ref); - rcu_read_lock(); - list_for_each_entry_rcu(found, head, list) - found->full = 0; - rcu_read_unlock(); + space_info = btrfs_find_space_info(fs_info, flags); + ASSERT(space_info); + percpu_counter_add_batch(&space_info->total_bytes_pinned, -ref->len, + BTRFS_TOTAL_BYTES_PINNED_BATCH); } /* simple helper to search for an existing data extent at a given offset */ @@ -1121,11 +1051,11 @@ static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset) __le64 lenum; lenum = cpu_to_le64(root_objectid); - high_crc = crc32c(high_crc, &lenum, sizeof(lenum)); + high_crc = btrfs_crc32c(high_crc, &lenum, sizeof(lenum)); lenum = cpu_to_le64(owner); - low_crc = crc32c(low_crc, &lenum, sizeof(lenum)); + low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); lenum = cpu_to_le64(offset); - low_crc = crc32c(low_crc, &lenum, sizeof(lenum)); + low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); return ((u64)high_crc << 31) ^ (u64)low_crc; } @@ -2065,7 +1995,7 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, btrfs_ref_tree_mod(fs_info, generic_ref); if (ret == 0 && old_ref_mod < 0 && new_ref_mod >= 0) - add_pinned_bytes(fs_info, generic_ref, -1); + sub_pinned_bytes(fs_info, generic_ref); return ret; } @@ -2462,7 +2392,7 @@ void btrfs_cleanup_ref_head_accounting(struct btrfs_fs_info *fs_info, flags = BTRFS_BLOCK_GROUP_SYSTEM; else flags = BTRFS_BLOCK_GROUP_METADATA; - space_info = __find_space_info(fs_info, flags); + space_info = btrfs_find_space_info(fs_info, flags); ASSERT(space_info); percpu_counter_add_batch(&space_info->total_bytes_pinned, -head->num_bytes, @@ -2824,49 +2754,6 @@ u64 btrfs_csum_bytes_to_leaves(struct btrfs_fs_info *fs_info, u64 csum_bytes) return num_csums; } -bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info) -{ - struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - bool ret = false; - u64 reserved; - - spin_lock(&global_rsv->lock); - reserved = global_rsv->reserved; - spin_unlock(&global_rsv->lock); - - /* - * Since the global reserve is just kind of magic we don't really want - * to rely on it to save our bacon, so if our size is more than the - * delayed_refs_rsv and the global rsv then it's time to think about - * bailing. - */ - spin_lock(&delayed_refs_rsv->lock); - reserved += delayed_refs_rsv->reserved; - if (delayed_refs_rsv->size >= reserved) - ret = true; - spin_unlock(&delayed_refs_rsv->lock); - return ret; -} - -int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans) -{ - u64 num_entries = - atomic_read(&trans->transaction->delayed_refs.num_entries); - u64 avg_runtime; - u64 val; - - smp_mb(); - avg_runtime = trans->fs_info->avg_delayed_ref_runtime; - val = num_entries * avg_runtime; - if (val >= NSEC_PER_SEC) - return 1; - if (val >= NSEC_PER_SEC / 2) - return 2; - - return btrfs_check_space_for_delayed_refs(trans->fs_info); -} - /* * this starts processing the delayed reference count updates and * extent insertions we have queued up so far. count can be @@ -3834,93 +3721,6 @@ void btrfs_wait_nocow_writers(struct btrfs_block_group_cache *bg) wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers)); } -static const char *alloc_name(u64 flags) -{ - switch (flags) { - case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA: - return "mixed"; - case BTRFS_BLOCK_GROUP_METADATA: - return "metadata"; - case BTRFS_BLOCK_GROUP_DATA: - return "data"; - case BTRFS_BLOCK_GROUP_SYSTEM: - return "system"; - default: - WARN_ON(1); - return "invalid-combination"; - }; -} - -static int create_space_info(struct btrfs_fs_info *info, u64 flags) -{ - - struct btrfs_space_info *space_info; - int i; - int ret; - - space_info = kzalloc(sizeof(*space_info), GFP_NOFS); - if (!space_info) - return -ENOMEM; - - ret = percpu_counter_init(&space_info->total_bytes_pinned, 0, - GFP_KERNEL); - if (ret) { - kfree(space_info); - return ret; - } - - for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) - INIT_LIST_HEAD(&space_info->block_groups[i]); - init_rwsem(&space_info->groups_sem); - spin_lock_init(&space_info->lock); - space_info->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK; - space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; - init_waitqueue_head(&space_info->wait); - INIT_LIST_HEAD(&space_info->ro_bgs); - INIT_LIST_HEAD(&space_info->tickets); - INIT_LIST_HEAD(&space_info->priority_tickets); - - ret = kobject_init_and_add(&space_info->kobj, &space_info_ktype, - info->space_info_kobj, "%s", - alloc_name(space_info->flags)); - if (ret) { - kobject_put(&space_info->kobj); - return ret; - } - - list_add_rcu(&space_info->list, &info->space_info); - if (flags & BTRFS_BLOCK_GROUP_DATA) - info->data_sinfo = space_info; - - return ret; -} - -static void update_space_info(struct btrfs_fs_info *info, u64 flags, - u64 total_bytes, u64 bytes_used, - u64 bytes_readonly, - struct btrfs_space_info **space_info) -{ - struct btrfs_space_info *found; - int factor; - - factor = btrfs_bg_type_to_factor(flags); - - found = __find_space_info(info, flags); - ASSERT(found); - spin_lock(&found->lock); - found->total_bytes += total_bytes; - found->disk_total += total_bytes * factor; - found->bytes_used += bytes_used; - found->disk_used += bytes_used * factor; - found->bytes_readonly += bytes_readonly; - if (total_bytes > 0) - found->full = 0; - space_info_add_new_bytes(info, found, total_bytes - - bytes_used - bytes_readonly); - spin_unlock(&found->lock); - *space_info = found; -} - static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) { u64 extra_flags = chunk_to_extended(flags) & @@ -4068,215 +3868,6 @@ u64 btrfs_system_alloc_profile(struct btrfs_fs_info *fs_info) return get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); } -static u64 btrfs_space_info_used(struct btrfs_space_info *s_info, - bool may_use_included) -{ - ASSERT(s_info); - return s_info->bytes_used + s_info->bytes_reserved + - s_info->bytes_pinned + s_info->bytes_readonly + - (may_use_included ? s_info->bytes_may_use : 0); -} - -int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes) -{ - struct btrfs_root *root = inode->root; - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_space_info *data_sinfo = fs_info->data_sinfo; - u64 used; - int ret = 0; - int need_commit = 2; - int have_pinned_space; - - /* make sure bytes are sectorsize aligned */ - bytes = ALIGN(bytes, fs_info->sectorsize); - - if (btrfs_is_free_space_inode(inode)) { - need_commit = 0; - ASSERT(current->journal_info); - } - -again: - /* make sure we have enough space to handle the data first */ - spin_lock(&data_sinfo->lock); - used = btrfs_space_info_used(data_sinfo, true); - - if (used + bytes > data_sinfo->total_bytes) { - struct btrfs_trans_handle *trans; - - /* - * if we don't have enough free bytes in this space then we need - * to alloc a new chunk. - */ - if (!data_sinfo->full) { - u64 alloc_target; - - data_sinfo->force_alloc = CHUNK_ALLOC_FORCE; - spin_unlock(&data_sinfo->lock); - - alloc_target = btrfs_data_alloc_profile(fs_info); - /* - * It is ugly that we don't call nolock join - * transaction for the free space inode case here. - * But it is safe because we only do the data space - * reservation for the free space cache in the - * transaction context, the common join transaction - * just increase the counter of the current transaction - * handler, doesn't try to acquire the trans_lock of - * the fs. - */ - trans = btrfs_join_transaction(root); - if (IS_ERR(trans)) - return PTR_ERR(trans); - - ret = do_chunk_alloc(trans, alloc_target, - CHUNK_ALLOC_NO_FORCE); - btrfs_end_transaction(trans); - if (ret < 0) { - if (ret != -ENOSPC) - return ret; - else { - have_pinned_space = 1; - goto commit_trans; - } - } - - goto again; - } - - /* - * If we don't have enough pinned space to deal with this - * allocation, and no removed chunk in current transaction, - * don't bother committing the transaction. - */ - have_pinned_space = __percpu_counter_compare( - &data_sinfo->total_bytes_pinned, - used + bytes - data_sinfo->total_bytes, - BTRFS_TOTAL_BYTES_PINNED_BATCH); - spin_unlock(&data_sinfo->lock); - - /* commit the current transaction and try again */ -commit_trans: - if (need_commit) { - need_commit--; - - if (need_commit > 0) { - btrfs_start_delalloc_roots(fs_info, -1); - btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, - (u64)-1); - } - - trans = btrfs_join_transaction(root); - if (IS_ERR(trans)) - return PTR_ERR(trans); - if (have_pinned_space >= 0 || - test_bit(BTRFS_TRANS_HAVE_FREE_BGS, - &trans->transaction->flags) || - need_commit > 0) { - ret = btrfs_commit_transaction(trans); - if (ret) - return ret; - /* - * The cleaner kthread might still be doing iput - * operations. Wait for it to finish so that - * more space is released. We don't need to - * explicitly run the delayed iputs here because - * the commit_transaction would have woken up - * the cleaner. - */ - ret = btrfs_wait_on_delayed_iputs(fs_info); - if (ret) - return ret; - goto again; - } else { - btrfs_end_transaction(trans); - } - } - - trace_btrfs_space_reservation(fs_info, - "space_info:enospc", - data_sinfo->flags, bytes, 1); - return -ENOSPC; - } - update_bytes_may_use(data_sinfo, bytes); - trace_btrfs_space_reservation(fs_info, "space_info", - data_sinfo->flags, bytes, 1); - spin_unlock(&data_sinfo->lock); - - return 0; -} - -int btrfs_check_data_free_space(struct inode *inode, - struct extent_changeset **reserved, u64 start, u64 len) -{ - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - int ret; - - /* align the range */ - len = round_up(start + len, fs_info->sectorsize) - - round_down(start, fs_info->sectorsize); - start = round_down(start, fs_info->sectorsize); - - ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode), len); - if (ret < 0) - return ret; - - /* Use new btrfs_qgroup_reserve_data to reserve precious data space. */ - ret = btrfs_qgroup_reserve_data(inode, reserved, start, len); - if (ret < 0) - btrfs_free_reserved_data_space_noquota(inode, start, len); - else - ret = 0; - return ret; -} - -/* - * Called if we need to clear a data reservation for this inode - * Normally in a error case. - * - * This one will *NOT* use accurate qgroup reserved space API, just for case - * which we can't sleep and is sure it won't affect qgroup reserved space. - * Like clear_bit_hook(). - */ -void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start, - u64 len) -{ - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct btrfs_space_info *data_sinfo; - - /* Make sure the range is aligned to sectorsize */ - len = round_up(start + len, fs_info->sectorsize) - - round_down(start, fs_info->sectorsize); - start = round_down(start, fs_info->sectorsize); - - data_sinfo = fs_info->data_sinfo; - spin_lock(&data_sinfo->lock); - update_bytes_may_use(data_sinfo, -len); - trace_btrfs_space_reservation(fs_info, "space_info", - data_sinfo->flags, len, 0); - spin_unlock(&data_sinfo->lock); -} - -/* - * Called if we need to clear a data reservation for this inode - * Normally in a error case. - * - * This one will handle the per-inode data rsv map for accurate reserved - * space framework. - */ -void btrfs_free_reserved_data_space(struct inode *inode, - struct extent_changeset *reserved, u64 start, u64 len) -{ - struct btrfs_root *root = BTRFS_I(inode)->root; - - /* Make sure the range is aligned to sectorsize */ - len = round_up(start + len, root->fs_info->sectorsize) - - round_down(start, root->fs_info->sectorsize); - start = round_down(start, root->fs_info->sectorsize); - - btrfs_free_reserved_data_space_noquota(inode, start, len); - btrfs_qgroup_free_data(inode, reserved, start, len); -} - static void force_metadata_allocation(struct btrfs_fs_info *info) { struct list_head *head = &info->space_info; @@ -4290,11 +3881,6 @@ static void force_metadata_allocation(struct btrfs_fs_info *info) rcu_read_unlock(); } -static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global) -{ - return (global->size << 1); -} - static int should_alloc_chunk(struct btrfs_fs_info *fs_info, struct btrfs_space_info *sinfo, int force) { @@ -4325,15 +3911,9 @@ static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type) { u64 num_dev; - if (type & (BTRFS_BLOCK_GROUP_RAID10 | - BTRFS_BLOCK_GROUP_RAID0 | - BTRFS_BLOCK_GROUP_RAID5 | - BTRFS_BLOCK_GROUP_RAID6)) + num_dev = btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)].devs_max; + if (!num_dev) num_dev = fs_info->fs_devices->rw_devices; - else if (type & BTRFS_BLOCK_GROUP_RAID1) - num_dev = 2; - else - num_dev = 1; /* DUP or single */ return num_dev; } @@ -4358,7 +3938,7 @@ void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) */ lockdep_assert_held(&fs_info->chunk_mutex); - info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); + info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); spin_lock(&info->lock); left = info->total_bytes - btrfs_space_info_used(info, true); spin_unlock(&info->lock); @@ -4372,7 +3952,7 @@ void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu", left, thresh, type); - dump_space_info(fs_info, info, 0, 0); + btrfs_dump_space_info(fs_info, info, 0, 0); } if (left < thresh) { @@ -4405,8 +3985,8 @@ void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) * - return 1 if it successfully allocates a chunk, * - return errors including -ENOSPC otherwise. */ -static int do_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, - int force) +int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, + enum btrfs_chunk_alloc_enum force) { struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_space_info *space_info; @@ -4418,7 +3998,7 @@ static int do_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, if (trans->allocating_chunk) return -ENOSPC; - space_info = __find_space_info(fs_info, flags); + space_info = btrfs_find_space_info(fs_info, flags); ASSERT(space_info); do { @@ -4525,1714 +4105,6 @@ out: return ret; } -static int can_overcommit(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, u64 bytes, - enum btrfs_reserve_flush_enum flush, - bool system_chunk) -{ - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - u64 profile; - u64 space_size; - u64 avail; - u64 used; - int factor; - - /* Don't overcommit when in mixed mode. */ - if (space_info->flags & BTRFS_BLOCK_GROUP_DATA) - return 0; - - if (system_chunk) - profile = btrfs_system_alloc_profile(fs_info); - else - profile = btrfs_metadata_alloc_profile(fs_info); - - used = btrfs_space_info_used(space_info, false); - - /* - * We only want to allow over committing if we have lots of actual space - * free, but if we don't have enough space to handle the global reserve - * space then we could end up having a real enospc problem when trying - * to allocate a chunk or some other such important allocation. - */ - spin_lock(&global_rsv->lock); - space_size = calc_global_rsv_need_space(global_rsv); - spin_unlock(&global_rsv->lock); - if (used + space_size >= space_info->total_bytes) - return 0; - - used += space_info->bytes_may_use; - - avail = atomic64_read(&fs_info->free_chunk_space); - - /* - * If we have dup, raid1 or raid10 then only half of the free - * space is actually usable. For raid56, the space info used - * doesn't include the parity drive, so we don't have to - * change the math - */ - factor = btrfs_bg_type_to_factor(profile); - avail = div_u64(avail, factor); - - /* - * If we aren't flushing all things, let us overcommit up to - * 1/2th of the space. If we can flush, don't let us overcommit - * too much, let it overcommit up to 1/8 of the space. - */ - if (flush == BTRFS_RESERVE_FLUSH_ALL) - avail >>= 3; - else - avail >>= 1; - - if (used + bytes < space_info->total_bytes + avail) - return 1; - return 0; -} - -static void btrfs_writeback_inodes_sb_nr(struct btrfs_fs_info *fs_info, - unsigned long nr_pages, int nr_items) -{ - struct super_block *sb = fs_info->sb; - - if (down_read_trylock(&sb->s_umount)) { - writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE); - up_read(&sb->s_umount); - } else { - /* - * We needn't worry the filesystem going from r/w to r/o though - * we don't acquire ->s_umount mutex, because the filesystem - * should guarantee the delalloc inodes list be empty after - * the filesystem is readonly(all dirty pages are written to - * the disk). - */ - btrfs_start_delalloc_roots(fs_info, nr_items); - if (!current->journal_info) - btrfs_wait_ordered_roots(fs_info, nr_items, 0, (u64)-1); - } -} - -static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info, - u64 to_reclaim) -{ - u64 bytes; - u64 nr; - - bytes = btrfs_calc_trans_metadata_size(fs_info, 1); - nr = div64_u64(to_reclaim, bytes); - if (!nr) - nr = 1; - return nr; -} - -#define EXTENT_SIZE_PER_ITEM SZ_256K - -/* - * shrink metadata reservation for delalloc - */ -static void shrink_delalloc(struct btrfs_fs_info *fs_info, u64 to_reclaim, - u64 orig, bool wait_ordered) -{ - struct btrfs_space_info *space_info; - struct btrfs_trans_handle *trans; - u64 delalloc_bytes; - u64 dio_bytes; - u64 async_pages; - u64 items; - long time_left; - unsigned long nr_pages; - int loops; - - /* Calc the number of the pages we need flush for space reservation */ - items = calc_reclaim_items_nr(fs_info, to_reclaim); - to_reclaim = items * EXTENT_SIZE_PER_ITEM; - - trans = (struct btrfs_trans_handle *)current->journal_info; - space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); - - delalloc_bytes = percpu_counter_sum_positive( - &fs_info->delalloc_bytes); - dio_bytes = percpu_counter_sum_positive(&fs_info->dio_bytes); - if (delalloc_bytes == 0 && dio_bytes == 0) { - if (trans) - return; - if (wait_ordered) - btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1); - return; - } - - /* - * If we are doing more ordered than delalloc we need to just wait on - * ordered extents, otherwise we'll waste time trying to flush delalloc - * that likely won't give us the space back we need. - */ - if (dio_bytes > delalloc_bytes) - wait_ordered = true; - - loops = 0; - while ((delalloc_bytes || dio_bytes) && loops < 3) { - nr_pages = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT; - - /* - * Triggers inode writeback for up to nr_pages. This will invoke - * ->writepages callback and trigger delalloc filling - * (btrfs_run_delalloc_range()). - */ - btrfs_writeback_inodes_sb_nr(fs_info, nr_pages, items); - - /* - * We need to wait for the compressed pages to start before - * we continue. - */ - async_pages = atomic_read(&fs_info->async_delalloc_pages); - if (!async_pages) - goto skip_async; - - /* - * Calculate how many compressed pages we want to be written - * before we continue. I.e if there are more async pages than we - * require wait_event will wait until nr_pages are written. - */ - if (async_pages <= nr_pages) - async_pages = 0; - else - async_pages -= nr_pages; - - wait_event(fs_info->async_submit_wait, - atomic_read(&fs_info->async_delalloc_pages) <= - (int)async_pages); -skip_async: - spin_lock(&space_info->lock); - if (list_empty(&space_info->tickets) && - list_empty(&space_info->priority_tickets)) { - spin_unlock(&space_info->lock); - break; - } - spin_unlock(&space_info->lock); - - loops++; - if (wait_ordered && !trans) { - btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1); - } else { - time_left = schedule_timeout_killable(1); - if (time_left) - break; - } - delalloc_bytes = percpu_counter_sum_positive( - &fs_info->delalloc_bytes); - dio_bytes = percpu_counter_sum_positive(&fs_info->dio_bytes); - } -} - -struct reserve_ticket { - u64 orig_bytes; - u64 bytes; - int error; - struct list_head list; - wait_queue_head_t wait; -}; - -/** - * maybe_commit_transaction - possibly commit the transaction if its ok to - * @root - the root we're allocating for - * @bytes - the number of bytes we want to reserve - * @force - force the commit - * - * This will check to make sure that committing the transaction will actually - * get us somewhere and then commit the transaction if it does. Otherwise it - * will return -ENOSPC. - */ -static int may_commit_transaction(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info) -{ - struct reserve_ticket *ticket = NULL; - struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv; - struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; - struct btrfs_trans_handle *trans; - u64 bytes_needed; - u64 reclaim_bytes = 0; - - trans = (struct btrfs_trans_handle *)current->journal_info; - if (trans) - return -EAGAIN; - - spin_lock(&space_info->lock); - if (!list_empty(&space_info->priority_tickets)) - ticket = list_first_entry(&space_info->priority_tickets, - struct reserve_ticket, list); - else if (!list_empty(&space_info->tickets)) - ticket = list_first_entry(&space_info->tickets, - struct reserve_ticket, list); - bytes_needed = (ticket) ? ticket->bytes : 0; - spin_unlock(&space_info->lock); - - if (!bytes_needed) - return 0; - - trans = btrfs_join_transaction(fs_info->extent_root); - if (IS_ERR(trans)) - return PTR_ERR(trans); - - /* - * See if there is enough pinned space to make this reservation, or if - * we have block groups that are going to be freed, allowing us to - * possibly do a chunk allocation the next loop through. - */ - if (test_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags) || - __percpu_counter_compare(&space_info->total_bytes_pinned, - bytes_needed, - BTRFS_TOTAL_BYTES_PINNED_BATCH) >= 0) - goto commit; - - /* - * See if there is some space in the delayed insertion reservation for - * this reservation. - */ - if (space_info != delayed_rsv->space_info) - goto enospc; - - spin_lock(&delayed_rsv->lock); - reclaim_bytes += delayed_rsv->reserved; - spin_unlock(&delayed_rsv->lock); - - spin_lock(&delayed_refs_rsv->lock); - reclaim_bytes += delayed_refs_rsv->reserved; - spin_unlock(&delayed_refs_rsv->lock); - if (reclaim_bytes >= bytes_needed) - goto commit; - bytes_needed -= reclaim_bytes; - - if (__percpu_counter_compare(&space_info->total_bytes_pinned, - bytes_needed, - BTRFS_TOTAL_BYTES_PINNED_BATCH) < 0) - goto enospc; - -commit: - return btrfs_commit_transaction(trans); -enospc: - btrfs_end_transaction(trans); - return -ENOSPC; -} - -/* - * Try to flush some data based on policy set by @state. This is only advisory - * and may fail for various reasons. The caller is supposed to examine the - * state of @space_info to detect the outcome. - */ -static void flush_space(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, u64 num_bytes, - int state) -{ - struct btrfs_root *root = fs_info->extent_root; - struct btrfs_trans_handle *trans; - int nr; - int ret = 0; - - switch (state) { - case FLUSH_DELAYED_ITEMS_NR: - case FLUSH_DELAYED_ITEMS: - if (state == FLUSH_DELAYED_ITEMS_NR) - nr = calc_reclaim_items_nr(fs_info, num_bytes) * 2; - else - nr = -1; - - trans = btrfs_join_transaction(root); - if (IS_ERR(trans)) { - ret = PTR_ERR(trans); - break; - } - ret = btrfs_run_delayed_items_nr(trans, nr); - btrfs_end_transaction(trans); - break; - case FLUSH_DELALLOC: - case FLUSH_DELALLOC_WAIT: - shrink_delalloc(fs_info, num_bytes * 2, num_bytes, - state == FLUSH_DELALLOC_WAIT); - break; - case FLUSH_DELAYED_REFS_NR: - case FLUSH_DELAYED_REFS: - trans = btrfs_join_transaction(root); - if (IS_ERR(trans)) { - ret = PTR_ERR(trans); - break; - } - if (state == FLUSH_DELAYED_REFS_NR) - nr = calc_reclaim_items_nr(fs_info, num_bytes); - else - nr = 0; - btrfs_run_delayed_refs(trans, nr); - btrfs_end_transaction(trans); - break; - case ALLOC_CHUNK: - case ALLOC_CHUNK_FORCE: - trans = btrfs_join_transaction(root); - if (IS_ERR(trans)) { - ret = PTR_ERR(trans); - break; - } - ret = do_chunk_alloc(trans, - btrfs_metadata_alloc_profile(fs_info), - (state == ALLOC_CHUNK) ? - CHUNK_ALLOC_NO_FORCE : CHUNK_ALLOC_FORCE); - btrfs_end_transaction(trans); - if (ret > 0 || ret == -ENOSPC) - ret = 0; - break; - case COMMIT_TRANS: - /* - * If we have pending delayed iputs then we could free up a - * bunch of pinned space, so make sure we run the iputs before - * we do our pinned bytes check below. - */ - btrfs_run_delayed_iputs(fs_info); - btrfs_wait_on_delayed_iputs(fs_info); - - ret = may_commit_transaction(fs_info, space_info); - break; - default: - ret = -ENOSPC; - break; - } - - trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, state, - ret); - return; -} - -static inline u64 -btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - bool system_chunk) -{ - struct reserve_ticket *ticket; - u64 used; - u64 expected; - u64 to_reclaim = 0; - - list_for_each_entry(ticket, &space_info->tickets, list) - to_reclaim += ticket->bytes; - list_for_each_entry(ticket, &space_info->priority_tickets, list) - to_reclaim += ticket->bytes; - if (to_reclaim) - return to_reclaim; - - to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M); - if (can_overcommit(fs_info, space_info, to_reclaim, - BTRFS_RESERVE_FLUSH_ALL, system_chunk)) - return 0; - - used = btrfs_space_info_used(space_info, true); - - if (can_overcommit(fs_info, space_info, SZ_1M, - BTRFS_RESERVE_FLUSH_ALL, system_chunk)) - expected = div_factor_fine(space_info->total_bytes, 95); - else - expected = div_factor_fine(space_info->total_bytes, 90); - - if (used > expected) - to_reclaim = used - expected; - else - to_reclaim = 0; - to_reclaim = min(to_reclaim, space_info->bytes_may_use + - space_info->bytes_reserved); - return to_reclaim; -} - -static inline int need_do_async_reclaim(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 used, bool system_chunk) -{ - u64 thresh = div_factor_fine(space_info->total_bytes, 98); - - /* If we're just plain full then async reclaim just slows us down. */ - if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) - return 0; - - if (!btrfs_calc_reclaim_metadata_size(fs_info, space_info, - system_chunk)) - return 0; - - return (used >= thresh && !btrfs_fs_closing(fs_info) && - !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); -} - -static bool wake_all_tickets(struct list_head *head) -{ - struct reserve_ticket *ticket; - - while (!list_empty(head)) { - ticket = list_first_entry(head, struct reserve_ticket, list); - list_del_init(&ticket->list); - ticket->error = -ENOSPC; - wake_up(&ticket->wait); - if (ticket->bytes != ticket->orig_bytes) - return true; - } - return false; -} - -/* - * This is for normal flushers, we can wait all goddamned day if we want to. We - * will loop and continuously try to flush as long as we are making progress. - * We count progress as clearing off tickets each time we have to loop. - */ -static void btrfs_async_reclaim_metadata_space(struct work_struct *work) -{ - struct btrfs_fs_info *fs_info; - struct btrfs_space_info *space_info; - u64 to_reclaim; - int flush_state; - int commit_cycles = 0; - u64 last_tickets_id; - - fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work); - space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); - - spin_lock(&space_info->lock); - to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info, - false); - if (!to_reclaim) { - space_info->flush = 0; - spin_unlock(&space_info->lock); - return; - } - last_tickets_id = space_info->tickets_id; - spin_unlock(&space_info->lock); - - flush_state = FLUSH_DELAYED_ITEMS_NR; - do { - flush_space(fs_info, space_info, to_reclaim, flush_state); - spin_lock(&space_info->lock); - if (list_empty(&space_info->tickets)) { - space_info->flush = 0; - spin_unlock(&space_info->lock); - return; - } - to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, - space_info, - false); - if (last_tickets_id == space_info->tickets_id) { - flush_state++; - } else { - last_tickets_id = space_info->tickets_id; - flush_state = FLUSH_DELAYED_ITEMS_NR; - if (commit_cycles) - commit_cycles--; - } - - /* - * We don't want to force a chunk allocation until we've tried - * pretty hard to reclaim space. Think of the case where we - * freed up a bunch of space and so have a lot of pinned space - * to reclaim. We would rather use that than possibly create a - * underutilized metadata chunk. So if this is our first run - * through the flushing state machine skip ALLOC_CHUNK_FORCE and - * commit the transaction. If nothing has changed the next go - * around then we can force a chunk allocation. - */ - if (flush_state == ALLOC_CHUNK_FORCE && !commit_cycles) - flush_state++; - - if (flush_state > COMMIT_TRANS) { - commit_cycles++; - if (commit_cycles > 2) { - if (wake_all_tickets(&space_info->tickets)) { - flush_state = FLUSH_DELAYED_ITEMS_NR; - commit_cycles--; - } else { - space_info->flush = 0; - } - } else { - flush_state = FLUSH_DELAYED_ITEMS_NR; - } - } - spin_unlock(&space_info->lock); - } while (flush_state <= COMMIT_TRANS); -} - -void btrfs_init_async_reclaim_work(struct work_struct *work) -{ - INIT_WORK(work, btrfs_async_reclaim_metadata_space); -} - -static const enum btrfs_flush_state priority_flush_states[] = { - FLUSH_DELAYED_ITEMS_NR, - FLUSH_DELAYED_ITEMS, - ALLOC_CHUNK, -}; - -static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - struct reserve_ticket *ticket) -{ - u64 to_reclaim; - int flush_state; - - spin_lock(&space_info->lock); - to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info, - false); - if (!to_reclaim) { - spin_unlock(&space_info->lock); - return; - } - spin_unlock(&space_info->lock); - - flush_state = 0; - do { - flush_space(fs_info, space_info, to_reclaim, - priority_flush_states[flush_state]); - flush_state++; - spin_lock(&space_info->lock); - if (ticket->bytes == 0) { - spin_unlock(&space_info->lock); - return; - } - spin_unlock(&space_info->lock); - } while (flush_state < ARRAY_SIZE(priority_flush_states)); -} - -static int wait_reserve_ticket(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - struct reserve_ticket *ticket) - -{ - DEFINE_WAIT(wait); - u64 reclaim_bytes = 0; - int ret = 0; - - spin_lock(&space_info->lock); - while (ticket->bytes > 0 && ticket->error == 0) { - ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE); - if (ret) { - ret = -EINTR; - break; - } - spin_unlock(&space_info->lock); - - schedule(); - - finish_wait(&ticket->wait, &wait); - spin_lock(&space_info->lock); - } - if (!ret) - ret = ticket->error; - if (!list_empty(&ticket->list)) - list_del_init(&ticket->list); - if (ticket->bytes && ticket->bytes < ticket->orig_bytes) - reclaim_bytes = ticket->orig_bytes - ticket->bytes; - spin_unlock(&space_info->lock); - - if (reclaim_bytes) - space_info_add_old_bytes(fs_info, space_info, reclaim_bytes); - return ret; -} - -/** - * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space - * @root - the root we're allocating for - * @space_info - the space info we want to allocate from - * @orig_bytes - the number of bytes we want - * @flush - whether or not we can flush to make our reservation - * - * This will reserve orig_bytes number of bytes from the space info associated - * with the block_rsv. If there is not enough space it will make an attempt to - * flush out space to make room. It will do this by flushing delalloc if - * possible or committing the transaction. If flush is 0 then no attempts to - * regain reservations will be made and this will fail if there is not enough - * space already. - */ -static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 orig_bytes, - enum btrfs_reserve_flush_enum flush, - bool system_chunk) -{ - struct reserve_ticket ticket; - u64 used; - u64 reclaim_bytes = 0; - int ret = 0; - - ASSERT(orig_bytes); - ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_ALL); - - spin_lock(&space_info->lock); - ret = -ENOSPC; - used = btrfs_space_info_used(space_info, true); - - /* - * If we have enough space then hooray, make our reservation and carry - * on. If not see if we can overcommit, and if we can, hooray carry on. - * If not things get more complicated. - */ - if (used + orig_bytes <= space_info->total_bytes) { - update_bytes_may_use(space_info, orig_bytes); - trace_btrfs_space_reservation(fs_info, "space_info", - space_info->flags, orig_bytes, 1); - ret = 0; - } else if (can_overcommit(fs_info, space_info, orig_bytes, flush, - system_chunk)) { - update_bytes_may_use(space_info, orig_bytes); - trace_btrfs_space_reservation(fs_info, "space_info", - space_info->flags, orig_bytes, 1); - ret = 0; - } - - /* - * If we couldn't make a reservation then setup our reservation ticket - * and kick the async worker if it's not already running. - * - * If we are a priority flusher then we just need to add our ticket to - * the list and we will do our own flushing further down. - */ - if (ret && flush != BTRFS_RESERVE_NO_FLUSH) { - ticket.orig_bytes = orig_bytes; - ticket.bytes = orig_bytes; - ticket.error = 0; - init_waitqueue_head(&ticket.wait); - if (flush == BTRFS_RESERVE_FLUSH_ALL) { - list_add_tail(&ticket.list, &space_info->tickets); - if (!space_info->flush) { - space_info->flush = 1; - trace_btrfs_trigger_flush(fs_info, - space_info->flags, - orig_bytes, flush, - "enospc"); - queue_work(system_unbound_wq, - &fs_info->async_reclaim_work); - } - } else { - list_add_tail(&ticket.list, - &space_info->priority_tickets); - } - } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { - used += orig_bytes; - /* - * We will do the space reservation dance during log replay, - * which means we won't have fs_info->fs_root set, so don't do - * the async reclaim as we will panic. - */ - if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && - need_do_async_reclaim(fs_info, space_info, - used, system_chunk) && - !work_busy(&fs_info->async_reclaim_work)) { - trace_btrfs_trigger_flush(fs_info, space_info->flags, - orig_bytes, flush, "preempt"); - queue_work(system_unbound_wq, - &fs_info->async_reclaim_work); - } - } - spin_unlock(&space_info->lock); - if (!ret || flush == BTRFS_RESERVE_NO_FLUSH) - return ret; - - if (flush == BTRFS_RESERVE_FLUSH_ALL) - return wait_reserve_ticket(fs_info, space_info, &ticket); - - ret = 0; - priority_reclaim_metadata_space(fs_info, space_info, &ticket); - spin_lock(&space_info->lock); - if (ticket.bytes) { - if (ticket.bytes < orig_bytes) - reclaim_bytes = orig_bytes - ticket.bytes; - list_del_init(&ticket.list); - ret = -ENOSPC; - } - spin_unlock(&space_info->lock); - - if (reclaim_bytes) - space_info_add_old_bytes(fs_info, space_info, reclaim_bytes); - ASSERT(list_empty(&ticket.list)); - return ret; -} - -/** - * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space - * @root - the root we're allocating for - * @block_rsv - the block_rsv we're allocating for - * @orig_bytes - the number of bytes we want - * @flush - whether or not we can flush to make our reservation - * - * This will reserve orig_bytes number of bytes from the space info associated - * with the block_rsv. If there is not enough space it will make an attempt to - * flush out space to make room. It will do this by flushing delalloc if - * possible or committing the transaction. If flush is 0 then no attempts to - * regain reservations will be made and this will fail if there is not enough - * space already. - */ -static int reserve_metadata_bytes(struct btrfs_root *root, - struct btrfs_block_rsv *block_rsv, - u64 orig_bytes, - enum btrfs_reserve_flush_enum flush) -{ - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - int ret; - bool system_chunk = (root == fs_info->chunk_root); - - ret = __reserve_metadata_bytes(fs_info, block_rsv->space_info, - orig_bytes, flush, system_chunk); - if (ret == -ENOSPC && - unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) { - if (block_rsv != global_rsv && - !block_rsv_use_bytes(global_rsv, orig_bytes)) - ret = 0; - } - if (ret == -ENOSPC) { - trace_btrfs_space_reservation(fs_info, "space_info:enospc", - block_rsv->space_info->flags, - orig_bytes, 1); - - if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) - dump_space_info(fs_info, block_rsv->space_info, - orig_bytes, 0); - } - return ret; -} - -static struct btrfs_block_rsv *get_block_rsv( - const struct btrfs_trans_handle *trans, - const struct btrfs_root *root) -{ - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_block_rsv *block_rsv = NULL; - - if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || - (root == fs_info->csum_root && trans->adding_csums) || - (root == fs_info->uuid_root)) - block_rsv = trans->block_rsv; - - if (!block_rsv) - block_rsv = root->block_rsv; - - if (!block_rsv) - block_rsv = &fs_info->empty_block_rsv; - - return block_rsv; -} - -static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, - u64 num_bytes) -{ - int ret = -ENOSPC; - spin_lock(&block_rsv->lock); - if (block_rsv->reserved >= num_bytes) { - block_rsv->reserved -= num_bytes; - if (block_rsv->reserved < block_rsv->size) - block_rsv->full = 0; - ret = 0; - } - spin_unlock(&block_rsv->lock); - return ret; -} - -static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, - u64 num_bytes, bool update_size) -{ - spin_lock(&block_rsv->lock); - block_rsv->reserved += num_bytes; - if (update_size) - block_rsv->size += num_bytes; - else if (block_rsv->reserved >= block_rsv->size) - block_rsv->full = 1; - spin_unlock(&block_rsv->lock); -} - -int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *dest, u64 num_bytes, - int min_factor) -{ - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - u64 min_bytes; - - if (global_rsv->space_info != dest->space_info) - return -ENOSPC; - - spin_lock(&global_rsv->lock); - min_bytes = div_factor(global_rsv->size, min_factor); - if (global_rsv->reserved < min_bytes + num_bytes) { - spin_unlock(&global_rsv->lock); - return -ENOSPC; - } - global_rsv->reserved -= num_bytes; - if (global_rsv->reserved < global_rsv->size) - global_rsv->full = 0; - spin_unlock(&global_rsv->lock); - - block_rsv_add_bytes(dest, num_bytes, true); - return 0; -} - -/** - * btrfs_migrate_to_delayed_refs_rsv - transfer bytes to our delayed refs rsv. - * @fs_info - the fs info for our fs. - * @src - the source block rsv to transfer from. - * @num_bytes - the number of bytes to transfer. - * - * This transfers up to the num_bytes amount from the src rsv to the - * delayed_refs_rsv. Any extra bytes are returned to the space info. - */ -void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *src, - u64 num_bytes) -{ - struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; - u64 to_free = 0; - - spin_lock(&src->lock); - src->reserved -= num_bytes; - src->size -= num_bytes; - spin_unlock(&src->lock); - - spin_lock(&delayed_refs_rsv->lock); - if (delayed_refs_rsv->size > delayed_refs_rsv->reserved) { - u64 delta = delayed_refs_rsv->size - - delayed_refs_rsv->reserved; - if (num_bytes > delta) { - to_free = num_bytes - delta; - num_bytes = delta; - } - } else { - to_free = num_bytes; - num_bytes = 0; - } - - if (num_bytes) - delayed_refs_rsv->reserved += num_bytes; - if (delayed_refs_rsv->reserved >= delayed_refs_rsv->size) - delayed_refs_rsv->full = 1; - spin_unlock(&delayed_refs_rsv->lock); - - if (num_bytes) - trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", - 0, num_bytes, 1); - if (to_free) - space_info_add_old_bytes(fs_info, delayed_refs_rsv->space_info, - to_free); -} - -/** - * btrfs_delayed_refs_rsv_refill - refill based on our delayed refs usage. - * @fs_info - the fs_info for our fs. - * @flush - control how we can flush for this reservation. - * - * This will refill the delayed block_rsv up to 1 items size worth of space and - * will return -ENOSPC if we can't make the reservation. - */ -int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info, - enum btrfs_reserve_flush_enum flush) -{ - struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv; - u64 limit = btrfs_calc_trans_metadata_size(fs_info, 1); - u64 num_bytes = 0; - int ret = -ENOSPC; - - spin_lock(&block_rsv->lock); - if (block_rsv->reserved < block_rsv->size) { - num_bytes = block_rsv->size - block_rsv->reserved; - num_bytes = min(num_bytes, limit); - } - spin_unlock(&block_rsv->lock); - - if (!num_bytes) - return 0; - - ret = reserve_metadata_bytes(fs_info->extent_root, block_rsv, - num_bytes, flush); - if (ret) - return ret; - block_rsv_add_bytes(block_rsv, num_bytes, 0); - trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", - 0, num_bytes, 1); - return 0; -} - -/* - * This is for space we already have accounted in space_info->bytes_may_use, so - * basically when we're returning space from block_rsv's. - */ -static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 num_bytes) -{ - struct reserve_ticket *ticket; - struct list_head *head; - u64 used; - enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH; - bool check_overcommit = false; - - spin_lock(&space_info->lock); - head = &space_info->priority_tickets; - - /* - * If we are over our limit then we need to check and see if we can - * overcommit, and if we can't then we just need to free up our space - * and not satisfy any requests. - */ - used = btrfs_space_info_used(space_info, true); - if (used - num_bytes >= space_info->total_bytes) - check_overcommit = true; -again: - while (!list_empty(head) && num_bytes) { - ticket = list_first_entry(head, struct reserve_ticket, - list); - /* - * We use 0 bytes because this space is already reserved, so - * adding the ticket space would be a double count. - */ - if (check_overcommit && - !can_overcommit(fs_info, space_info, 0, flush, false)) - break; - if (num_bytes >= ticket->bytes) { - list_del_init(&ticket->list); - num_bytes -= ticket->bytes; - ticket->bytes = 0; - space_info->tickets_id++; - wake_up(&ticket->wait); - } else { - ticket->bytes -= num_bytes; - num_bytes = 0; - } - } - - if (num_bytes && head == &space_info->priority_tickets) { - head = &space_info->tickets; - flush = BTRFS_RESERVE_FLUSH_ALL; - goto again; - } - update_bytes_may_use(space_info, -num_bytes); - trace_btrfs_space_reservation(fs_info, "space_info", - space_info->flags, num_bytes, 0); - spin_unlock(&space_info->lock); -} - -/* - * This is for newly allocated space that isn't accounted in - * space_info->bytes_may_use yet. So if we allocate a chunk or unpin an extent - * we use this helper. - */ -static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - u64 num_bytes) -{ - struct reserve_ticket *ticket; - struct list_head *head = &space_info->priority_tickets; - -again: - while (!list_empty(head) && num_bytes) { - ticket = list_first_entry(head, struct reserve_ticket, - list); - if (num_bytes >= ticket->bytes) { - trace_btrfs_space_reservation(fs_info, "space_info", - space_info->flags, - ticket->bytes, 1); - list_del_init(&ticket->list); - num_bytes -= ticket->bytes; - update_bytes_may_use(space_info, ticket->bytes); - ticket->bytes = 0; - space_info->tickets_id++; - wake_up(&ticket->wait); - } else { - trace_btrfs_space_reservation(fs_info, "space_info", - space_info->flags, - num_bytes, 1); - update_bytes_may_use(space_info, num_bytes); - ticket->bytes -= num_bytes; - num_bytes = 0; - } - } - - if (num_bytes && head == &space_info->priority_tickets) { - head = &space_info->tickets; - goto again; - } -} - -static u64 block_rsv_release_bytes(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *block_rsv, - struct btrfs_block_rsv *dest, u64 num_bytes, - u64 *qgroup_to_release_ret) -{ - struct btrfs_space_info *space_info = block_rsv->space_info; - u64 qgroup_to_release = 0; - u64 ret; - - spin_lock(&block_rsv->lock); - if (num_bytes == (u64)-1) { - num_bytes = block_rsv->size; - qgroup_to_release = block_rsv->qgroup_rsv_size; - } - block_rsv->size -= num_bytes; - if (block_rsv->reserved >= block_rsv->size) { - num_bytes = block_rsv->reserved - block_rsv->size; - block_rsv->reserved = block_rsv->size; - block_rsv->full = 1; - } else { - num_bytes = 0; - } - if (block_rsv->qgroup_rsv_reserved >= block_rsv->qgroup_rsv_size) { - qgroup_to_release = block_rsv->qgroup_rsv_reserved - - block_rsv->qgroup_rsv_size; - block_rsv->qgroup_rsv_reserved = block_rsv->qgroup_rsv_size; - } else { - qgroup_to_release = 0; - } - spin_unlock(&block_rsv->lock); - - ret = num_bytes; - if (num_bytes > 0) { - if (dest) { - spin_lock(&dest->lock); - if (!dest->full) { - u64 bytes_to_add; - - bytes_to_add = dest->size - dest->reserved; - bytes_to_add = min(num_bytes, bytes_to_add); - dest->reserved += bytes_to_add; - if (dest->reserved >= dest->size) - dest->full = 1; - num_bytes -= bytes_to_add; - } - spin_unlock(&dest->lock); - } - if (num_bytes) - space_info_add_old_bytes(fs_info, space_info, - num_bytes); - } - if (qgroup_to_release_ret) - *qgroup_to_release_ret = qgroup_to_release; - return ret; -} - -int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src, - struct btrfs_block_rsv *dst, u64 num_bytes, - bool update_size) -{ - int ret; - - ret = block_rsv_use_bytes(src, num_bytes); - if (ret) - return ret; - - block_rsv_add_bytes(dst, num_bytes, update_size); - return 0; -} - -void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type) -{ - memset(rsv, 0, sizeof(*rsv)); - spin_lock_init(&rsv->lock); - rsv->type = type; -} - -void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *rsv, - unsigned short type) -{ - btrfs_init_block_rsv(rsv, type); - rsv->space_info = __find_space_info(fs_info, - BTRFS_BLOCK_GROUP_METADATA); -} - -struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, - unsigned short type) -{ - struct btrfs_block_rsv *block_rsv; - - block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS); - if (!block_rsv) - return NULL; - - btrfs_init_metadata_block_rsv(fs_info, block_rsv, type); - return block_rsv; -} - -void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *rsv) -{ - if (!rsv) - return; - btrfs_block_rsv_release(fs_info, rsv, (u64)-1); - kfree(rsv); -} - -int btrfs_block_rsv_add(struct btrfs_root *root, - struct btrfs_block_rsv *block_rsv, u64 num_bytes, - enum btrfs_reserve_flush_enum flush) -{ - int ret; - - if (num_bytes == 0) - return 0; - - ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush); - if (!ret) - block_rsv_add_bytes(block_rsv, num_bytes, true); - - return ret; -} - -int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor) -{ - u64 num_bytes = 0; - int ret = -ENOSPC; - - if (!block_rsv) - return 0; - - spin_lock(&block_rsv->lock); - num_bytes = div_factor(block_rsv->size, min_factor); - if (block_rsv->reserved >= num_bytes) - ret = 0; - spin_unlock(&block_rsv->lock); - - return ret; -} - -int btrfs_block_rsv_refill(struct btrfs_root *root, - struct btrfs_block_rsv *block_rsv, u64 min_reserved, - enum btrfs_reserve_flush_enum flush) -{ - u64 num_bytes = 0; - int ret = -ENOSPC; - - if (!block_rsv) - return 0; - - spin_lock(&block_rsv->lock); - num_bytes = min_reserved; - if (block_rsv->reserved >= num_bytes) - ret = 0; - else - num_bytes -= block_rsv->reserved; - spin_unlock(&block_rsv->lock); - - if (!ret) - return 0; - - ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush); - if (!ret) { - block_rsv_add_bytes(block_rsv, num_bytes, false); - return 0; - } - - return ret; -} - -static u64 __btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *block_rsv, - u64 num_bytes, u64 *qgroup_to_release) -{ - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; - struct btrfs_block_rsv *target = delayed_rsv; - - if (target->full || target == block_rsv) - target = global_rsv; - - if (block_rsv->space_info != target->space_info) - target = NULL; - - return block_rsv_release_bytes(fs_info, block_rsv, target, num_bytes, - qgroup_to_release); -} - -void btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *block_rsv, - u64 num_bytes) -{ - __btrfs_block_rsv_release(fs_info, block_rsv, num_bytes, NULL); -} - -/** - * btrfs_inode_rsv_release - release any excessive reservation. - * @inode - the inode we need to release from. - * @qgroup_free - free or convert qgroup meta. - * Unlike normal operation, qgroup meta reservation needs to know if we are - * freeing qgroup reservation or just converting it into per-trans. Normally - * @qgroup_free is true for error handling, and false for normal release. - * - * This is the same as btrfs_block_rsv_release, except that it handles the - * tracepoint for the reservation. - */ -static void btrfs_inode_rsv_release(struct btrfs_inode *inode, bool qgroup_free) -{ - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct btrfs_block_rsv *block_rsv = &inode->block_rsv; - u64 released = 0; - u64 qgroup_to_release = 0; - - /* - * Since we statically set the block_rsv->size we just want to say we - * are releasing 0 bytes, and then we'll just get the reservation over - * the size free'd. - */ - released = __btrfs_block_rsv_release(fs_info, block_rsv, 0, - &qgroup_to_release); - if (released > 0) - trace_btrfs_space_reservation(fs_info, "delalloc", - btrfs_ino(inode), released, 0); - if (qgroup_free) - btrfs_qgroup_free_meta_prealloc(inode->root, qgroup_to_release); - else - btrfs_qgroup_convert_reserved_meta(inode->root, - qgroup_to_release); -} - -/** - * btrfs_delayed_refs_rsv_release - release a ref head's reservation. - * @fs_info - the fs_info for our fs. - * @nr - the number of items to drop. - * - * This drops the delayed ref head's count from the delayed refs rsv and frees - * any excess reservation we had. - */ -void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr) -{ - struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv; - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - u64 num_bytes = btrfs_calc_trans_metadata_size(fs_info, nr); - u64 released = 0; - - released = block_rsv_release_bytes(fs_info, block_rsv, global_rsv, - num_bytes, NULL); - if (released) - trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", - 0, released, 0); -} - -static void update_global_block_rsv(struct btrfs_fs_info *fs_info) -{ - struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; - struct btrfs_space_info *sinfo = block_rsv->space_info; - u64 num_bytes; - - /* - * The global block rsv is based on the size of the extent tree, the - * checksum tree and the root tree. If the fs is empty we want to set - * it to a minimal amount for safety. - */ - num_bytes = btrfs_root_used(&fs_info->extent_root->root_item) + - btrfs_root_used(&fs_info->csum_root->root_item) + - btrfs_root_used(&fs_info->tree_root->root_item); - num_bytes = max_t(u64, num_bytes, SZ_16M); - - spin_lock(&sinfo->lock); - spin_lock(&block_rsv->lock); - - block_rsv->size = min_t(u64, num_bytes, SZ_512M); - - if (block_rsv->reserved < block_rsv->size) { - num_bytes = btrfs_space_info_used(sinfo, true); - if (sinfo->total_bytes > num_bytes) { - num_bytes = sinfo->total_bytes - num_bytes; - num_bytes = min(num_bytes, - block_rsv->size - block_rsv->reserved); - block_rsv->reserved += num_bytes; - update_bytes_may_use(sinfo, num_bytes); - trace_btrfs_space_reservation(fs_info, "space_info", - sinfo->flags, num_bytes, - 1); - } - } else if (block_rsv->reserved > block_rsv->size) { - num_bytes = block_rsv->reserved - block_rsv->size; - update_bytes_may_use(sinfo, -num_bytes); - trace_btrfs_space_reservation(fs_info, "space_info", - sinfo->flags, num_bytes, 0); - block_rsv->reserved = block_rsv->size; - } - - if (block_rsv->reserved == block_rsv->size) - block_rsv->full = 1; - else - block_rsv->full = 0; - - spin_unlock(&block_rsv->lock); - spin_unlock(&sinfo->lock); -} - -static void init_global_block_rsv(struct btrfs_fs_info *fs_info) -{ - struct btrfs_space_info *space_info; - - space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); - fs_info->chunk_block_rsv.space_info = space_info; - - space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); - fs_info->global_block_rsv.space_info = space_info; - fs_info->trans_block_rsv.space_info = space_info; - fs_info->empty_block_rsv.space_info = space_info; - fs_info->delayed_block_rsv.space_info = space_info; - fs_info->delayed_refs_rsv.space_info = space_info; - - fs_info->extent_root->block_rsv = &fs_info->delayed_refs_rsv; - fs_info->csum_root->block_rsv = &fs_info->delayed_refs_rsv; - fs_info->dev_root->block_rsv = &fs_info->global_block_rsv; - fs_info->tree_root->block_rsv = &fs_info->global_block_rsv; - if (fs_info->quota_root) - fs_info->quota_root->block_rsv = &fs_info->global_block_rsv; - fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv; - - update_global_block_rsv(fs_info); -} - -static void release_global_block_rsv(struct btrfs_fs_info *fs_info) -{ - block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL, - (u64)-1, NULL); - WARN_ON(fs_info->trans_block_rsv.size > 0); - WARN_ON(fs_info->trans_block_rsv.reserved > 0); - WARN_ON(fs_info->chunk_block_rsv.size > 0); - WARN_ON(fs_info->chunk_block_rsv.reserved > 0); - WARN_ON(fs_info->delayed_block_rsv.size > 0); - WARN_ON(fs_info->delayed_block_rsv.reserved > 0); - WARN_ON(fs_info->delayed_refs_rsv.reserved > 0); - WARN_ON(fs_info->delayed_refs_rsv.size > 0); -} - -/* - * btrfs_update_delayed_refs_rsv - adjust the size of the delayed refs rsv - * @trans - the trans that may have generated delayed refs - * - * This is to be called anytime we may have adjusted trans->delayed_ref_updates, - * it'll calculate the additional size and add it to the delayed_refs_rsv. - */ -void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans) -{ - struct btrfs_fs_info *fs_info = trans->fs_info; - struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; - u64 num_bytes; - - if (!trans->delayed_ref_updates) - return; - - num_bytes = btrfs_calc_trans_metadata_size(fs_info, - trans->delayed_ref_updates); - spin_lock(&delayed_rsv->lock); - delayed_rsv->size += num_bytes; - delayed_rsv->full = 0; - spin_unlock(&delayed_rsv->lock); - trans->delayed_ref_updates = 0; -} - -/* - * To be called after all the new block groups attached to the transaction - * handle have been created (btrfs_create_pending_block_groups()). - */ -void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans) -{ - struct btrfs_fs_info *fs_info = trans->fs_info; - - if (!trans->chunk_bytes_reserved) - return; - - WARN_ON_ONCE(!list_empty(&trans->new_bgs)); - - block_rsv_release_bytes(fs_info, &fs_info->chunk_block_rsv, NULL, - trans->chunk_bytes_reserved, NULL); - trans->chunk_bytes_reserved = 0; -} - -/* - * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation - * root: the root of the parent directory - * rsv: block reservation - * items: the number of items that we need do reservation - * use_global_rsv: allow fallback to the global block reservation - * - * This function is used to reserve the space for snapshot/subvolume - * creation and deletion. Those operations are different with the - * common file/directory operations, they change two fs/file trees - * and root tree, the number of items that the qgroup reserves is - * different with the free space reservation. So we can not use - * the space reservation mechanism in start_transaction(). - */ -int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, - struct btrfs_block_rsv *rsv, int items, - bool use_global_rsv) -{ - u64 qgroup_num_bytes = 0; - u64 num_bytes; - int ret; - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - - if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) { - /* One for parent inode, two for dir entries */ - qgroup_num_bytes = 3 * fs_info->nodesize; - ret = btrfs_qgroup_reserve_meta_prealloc(root, - qgroup_num_bytes, true); - if (ret) - return ret; - } - - num_bytes = btrfs_calc_trans_metadata_size(fs_info, items); - rsv->space_info = __find_space_info(fs_info, - BTRFS_BLOCK_GROUP_METADATA); - ret = btrfs_block_rsv_add(root, rsv, num_bytes, - BTRFS_RESERVE_FLUSH_ALL); - - if (ret == -ENOSPC && use_global_rsv) - ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes, true); - - if (ret && qgroup_num_bytes) - btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes); - - return ret; -} - -void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *rsv) -{ - btrfs_block_rsv_release(fs_info, rsv, (u64)-1); -} - -static void btrfs_calculate_inode_block_rsv_size(struct btrfs_fs_info *fs_info, - struct btrfs_inode *inode) -{ - struct btrfs_block_rsv *block_rsv = &inode->block_rsv; - u64 reserve_size = 0; - u64 qgroup_rsv_size = 0; - u64 csum_leaves; - unsigned outstanding_extents; - - lockdep_assert_held(&inode->lock); - outstanding_extents = inode->outstanding_extents; - if (outstanding_extents) - reserve_size = btrfs_calc_trans_metadata_size(fs_info, - outstanding_extents + 1); - csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, - inode->csum_bytes); - reserve_size += btrfs_calc_trans_metadata_size(fs_info, - csum_leaves); - /* - * For qgroup rsv, the calculation is very simple: - * account one nodesize for each outstanding extent - * - * This is overestimating in most cases. - */ - qgroup_rsv_size = (u64)outstanding_extents * fs_info->nodesize; - - spin_lock(&block_rsv->lock); - block_rsv->size = reserve_size; - block_rsv->qgroup_rsv_size = qgroup_rsv_size; - spin_unlock(&block_rsv->lock); -} - -static void calc_inode_reservations(struct btrfs_fs_info *fs_info, - u64 num_bytes, u64 *meta_reserve, - u64 *qgroup_reserve) -{ - u64 nr_extents = count_max_extents(num_bytes); - u64 csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, num_bytes); - - /* We add one for the inode update at finish ordered time */ - *meta_reserve = btrfs_calc_trans_metadata_size(fs_info, - nr_extents + csum_leaves + 1); - *qgroup_reserve = nr_extents * fs_info->nodesize; -} - -int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes) -{ - struct btrfs_root *root = inode->root; - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_block_rsv *block_rsv = &inode->block_rsv; - u64 meta_reserve, qgroup_reserve; - unsigned nr_extents; - enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL; - int ret = 0; - bool delalloc_lock = true; - - /* If we are a free space inode we need to not flush since we will be in - * the middle of a transaction commit. We also don't need the delalloc - * mutex since we won't race with anybody. We need this mostly to make - * lockdep shut its filthy mouth. - * - * If we have a transaction open (can happen if we call truncate_block - * from truncate), then we need FLUSH_LIMIT so we don't deadlock. - */ - if (btrfs_is_free_space_inode(inode)) { - flush = BTRFS_RESERVE_NO_FLUSH; - delalloc_lock = false; - } else { - if (current->journal_info) - flush = BTRFS_RESERVE_FLUSH_LIMIT; - - if (btrfs_transaction_in_commit(fs_info)) - schedule_timeout(1); - } - - if (delalloc_lock) - mutex_lock(&inode->delalloc_mutex); - - num_bytes = ALIGN(num_bytes, fs_info->sectorsize); - - /* - * We always want to do it this way, every other way is wrong and ends - * in tears. Pre-reserving the amount we are going to add will always - * be the right way, because otherwise if we have enough parallelism we - * could end up with thousands of inodes all holding little bits of - * reservations they were able to make previously and the only way to - * reclaim that space is to ENOSPC out the operations and clear - * everything out and try again, which is bad. This way we just - * over-reserve slightly, and clean up the mess when we are done. - */ - calc_inode_reservations(fs_info, num_bytes, &meta_reserve, - &qgroup_reserve); - ret = btrfs_qgroup_reserve_meta_prealloc(root, qgroup_reserve, true); - if (ret) - goto out_fail; - ret = reserve_metadata_bytes(root, block_rsv, meta_reserve, flush); - if (ret) - goto out_qgroup; - - /* - * Now we need to update our outstanding extents and csum bytes _first_ - * and then add the reservation to the block_rsv. This keeps us from - * racing with an ordered completion or some such that would think it - * needs to free the reservation we just made. - */ - spin_lock(&inode->lock); - nr_extents = count_max_extents(num_bytes); - btrfs_mod_outstanding_extents(inode, nr_extents); - inode->csum_bytes += num_bytes; - btrfs_calculate_inode_block_rsv_size(fs_info, inode); - spin_unlock(&inode->lock); - - /* Now we can safely add our space to our block rsv */ - block_rsv_add_bytes(block_rsv, meta_reserve, false); - trace_btrfs_space_reservation(root->fs_info, "delalloc", - btrfs_ino(inode), meta_reserve, 1); - - spin_lock(&block_rsv->lock); - block_rsv->qgroup_rsv_reserved += qgroup_reserve; - spin_unlock(&block_rsv->lock); - - if (delalloc_lock) - mutex_unlock(&inode->delalloc_mutex); - return 0; -out_qgroup: - btrfs_qgroup_free_meta_prealloc(root, qgroup_reserve); -out_fail: - btrfs_inode_rsv_release(inode, true); - if (delalloc_lock) - mutex_unlock(&inode->delalloc_mutex); - return ret; -} - -/** - * btrfs_delalloc_release_metadata - release a metadata reservation for an inode - * @inode: the inode to release the reservation for. - * @num_bytes: the number of bytes we are releasing. - * @qgroup_free: free qgroup reservation or convert it to per-trans reservation - * - * This will release the metadata reservation for an inode. This can be called - * once we complete IO for a given set of bytes to release their metadata - * reservations, or on error for the same reason. - */ -void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes, - bool qgroup_free) -{ - struct btrfs_fs_info *fs_info = inode->root->fs_info; - - num_bytes = ALIGN(num_bytes, fs_info->sectorsize); - spin_lock(&inode->lock); - inode->csum_bytes -= num_bytes; - btrfs_calculate_inode_block_rsv_size(fs_info, inode); - spin_unlock(&inode->lock); - - if (btrfs_is_testing(fs_info)) - return; - - btrfs_inode_rsv_release(inode, qgroup_free); -} - -/** - * btrfs_delalloc_release_extents - release our outstanding_extents - * @inode: the inode to balance the reservation for. - * @num_bytes: the number of bytes we originally reserved with - * @qgroup_free: do we need to free qgroup meta reservation or convert them. - * - * When we reserve space we increase outstanding_extents for the extents we may - * add. Once we've set the range as delalloc or created our ordered extents we - * have outstanding_extents to track the real usage, so we use this to free our - * temporarily tracked outstanding_extents. This _must_ be used in conjunction - * with btrfs_delalloc_reserve_metadata. - */ -void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes, - bool qgroup_free) -{ - struct btrfs_fs_info *fs_info = inode->root->fs_info; - unsigned num_extents; - - spin_lock(&inode->lock); - num_extents = count_max_extents(num_bytes); - btrfs_mod_outstanding_extents(inode, -num_extents); - btrfs_calculate_inode_block_rsv_size(fs_info, inode); - spin_unlock(&inode->lock); - - if (btrfs_is_testing(fs_info)) - return; - - btrfs_inode_rsv_release(inode, qgroup_free); -} - -/** - * btrfs_delalloc_reserve_space - reserve data and metadata space for - * delalloc - * @inode: inode we're writing to - * @start: start range we are writing to - * @len: how long the range we are writing to - * @reserved: mandatory parameter, record actually reserved qgroup ranges of - * current reservation. - * - * This will do the following things - * - * o reserve space in data space info for num bytes - * and reserve precious corresponding qgroup space - * (Done in check_data_free_space) - * - * o reserve space for metadata space, based on the number of outstanding - * extents and how much csums will be needed - * also reserve metadata space in a per root over-reserve method. - * o add to the inodes->delalloc_bytes - * o add it to the fs_info's delalloc inodes list. - * (Above 3 all done in delalloc_reserve_metadata) - * - * Return 0 for success - * Return <0 for error(-ENOSPC or -EQUOT) - */ -int btrfs_delalloc_reserve_space(struct inode *inode, - struct extent_changeset **reserved, u64 start, u64 len) -{ - int ret; - - ret = btrfs_check_data_free_space(inode, reserved, start, len); - if (ret < 0) - return ret; - ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len); - if (ret < 0) - btrfs_free_reserved_data_space(inode, *reserved, start, len); - return ret; -} - -/** - * btrfs_delalloc_release_space - release data and metadata space for delalloc - * @inode: inode we're releasing space for - * @start: start position of the space already reserved - * @len: the len of the space already reserved - * @release_bytes: the len of the space we consumed or didn't use - * - * This function will release the metadata space that was not used and will - * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes - * list if there are no delalloc bytes left. - * Also it will handle the qgroup reserved space. - */ -void btrfs_delalloc_release_space(struct inode *inode, - struct extent_changeset *reserved, - u64 start, u64 len, bool qgroup_free) -{ - btrfs_delalloc_release_metadata(BTRFS_I(inode), len, qgroup_free); - btrfs_free_reserved_data_space(inode, reserved, start, len); -} - static int update_block_group(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, int alloc) { @@ -6296,7 +4168,8 @@ static int update_block_group(struct btrfs_trans_handle *trans, old_val -= num_bytes; btrfs_set_block_group_used(&cache->item, old_val); cache->pinned += num_bytes; - update_bytes_pinned(cache->space_info, num_bytes); + btrfs_space_info_update_bytes_pinned(info, + cache->space_info, num_bytes); cache->space_info->bytes_used -= num_bytes; cache->space_info->disk_used -= num_bytes * factor; spin_unlock(&cache->lock); @@ -6371,7 +4244,8 @@ static int pin_down_extent(struct btrfs_block_group_cache *cache, spin_lock(&cache->space_info->lock); spin_lock(&cache->lock); cache->pinned += num_bytes; - update_bytes_pinned(cache->space_info, num_bytes); + btrfs_space_info_update_bytes_pinned(fs_info, cache->space_info, + num_bytes); if (reserved) { cache->reserved -= num_bytes; cache->space_info->bytes_reserved -= num_bytes; @@ -6580,7 +4454,8 @@ static int btrfs_add_reserved_bytes(struct btrfs_block_group_cache *cache, } else { cache->reserved += num_bytes; space_info->bytes_reserved += num_bytes; - update_bytes_may_use(space_info, -ram_bytes); + btrfs_space_info_update_bytes_may_use(cache->fs_info, + space_info, -ram_bytes); if (delalloc) cache->delalloc_bytes += num_bytes; } @@ -6646,7 +4521,7 @@ void btrfs_prepare_extent_commit(struct btrfs_fs_info *fs_info) up_write(&fs_info->commit_root_sem); - update_global_block_rsv(fs_info); + btrfs_update_global_block_rsv(fs_info); } /* @@ -6736,7 +4611,7 @@ static int unpin_extent_range(struct btrfs_fs_info *fs_info, spin_lock(&space_info->lock); spin_lock(&cache->lock); cache->pinned -= len; - update_bytes_pinned(space_info, -len); + btrfs_space_info_update_bytes_pinned(fs_info, space_info, -len); trace_btrfs_space_reservation(fs_info, "pinned", space_info->flags, len, 0); @@ -6757,7 +4632,8 @@ static int unpin_extent_range(struct btrfs_fs_info *fs_info, to_add = min(len, global_rsv->size - global_rsv->reserved); global_rsv->reserved += to_add; - update_bytes_may_use(space_info, to_add); + btrfs_space_info_update_bytes_may_use(fs_info, + space_info, to_add); if (global_rsv->reserved >= global_rsv->size) global_rsv->full = 1; trace_btrfs_space_reservation(fs_info, @@ -6769,8 +4645,8 @@ static int unpin_extent_range(struct btrfs_fs_info *fs_info, spin_unlock(&global_rsv->lock); /* Add to any tickets we may have */ if (len) - space_info_add_new_bytes(fs_info, space_info, - len); + btrfs_space_info_add_new_bytes(fs_info, + space_info, len); } spin_unlock(&space_info->lock); } @@ -7191,7 +5067,7 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, } out: if (pin) - add_pinned_bytes(fs_info, &generic_ref, 1); + add_pinned_bytes(fs_info, &generic_ref); if (last_ref) { /* @@ -7239,7 +5115,7 @@ int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_ref *ref) btrfs_ref_tree_mod(fs_info, ref); if (ret == 0 && old_ref_mod >= 0 && new_ref_mod < 0) - add_pinned_bytes(fs_info, ref, 1); + add_pinned_bytes(fs_info, ref); return ret; } @@ -7292,10 +5168,10 @@ wait_block_group_cache_done(struct btrfs_block_group_cache *cache) } enum btrfs_loop_type { - LOOP_CACHING_NOWAIT = 0, - LOOP_CACHING_WAIT = 1, - LOOP_ALLOC_CHUNK = 2, - LOOP_NO_EMPTY_SIZE = 3, + LOOP_CACHING_NOWAIT, + LOOP_CACHING_WAIT, + LOOP_ALLOC_CHUNK, + LOOP_NO_EMPTY_SIZE, }; static inline void @@ -7661,8 +5537,8 @@ static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info, return ret; } - ret = do_chunk_alloc(trans, ffe_ctl->flags, - CHUNK_ALLOC_FORCE); + ret = btrfs_chunk_alloc(trans, ffe_ctl->flags, + CHUNK_ALLOC_FORCE); /* * If we can't allocate a new chunk we've already looped @@ -7758,7 +5634,7 @@ static noinline int find_free_extent(struct btrfs_fs_info *fs_info, trace_find_free_extent(fs_info, num_bytes, empty_size, flags); - space_info = __find_space_info(fs_info, flags); + space_info = btrfs_find_space_info(fs_info, flags); if (!space_info) { btrfs_err(fs_info, "No space info for %llu", flags); return -ENOSPC; @@ -7863,9 +5739,8 @@ search: */ if (!block_group_bits(block_group, flags)) { u64 extra = BTRFS_BLOCK_GROUP_DUP | - BTRFS_BLOCK_GROUP_RAID1 | - BTRFS_BLOCK_GROUP_RAID5 | - BTRFS_BLOCK_GROUP_RAID6 | + BTRFS_BLOCK_GROUP_RAID1_MASK | + BTRFS_BLOCK_GROUP_RAID56_MASK | BTRFS_BLOCK_GROUP_RAID10; /* @@ -7984,60 +5859,6 @@ loop: return ret; } -#define DUMP_BLOCK_RSV(fs_info, rsv_name) \ -do { \ - struct btrfs_block_rsv *__rsv = &(fs_info)->rsv_name; \ - spin_lock(&__rsv->lock); \ - btrfs_info(fs_info, #rsv_name ": size %llu reserved %llu", \ - __rsv->size, __rsv->reserved); \ - spin_unlock(&__rsv->lock); \ -} while (0) - -static void dump_space_info(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *info, u64 bytes, - int dump_block_groups) -{ - struct btrfs_block_group_cache *cache; - int index = 0; - - spin_lock(&info->lock); - btrfs_info(fs_info, "space_info %llu has %llu free, is %sfull", - info->flags, - info->total_bytes - btrfs_space_info_used(info, true), - info->full ? "" : "not "); - btrfs_info(fs_info, - "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu", - info->total_bytes, info->bytes_used, info->bytes_pinned, - info->bytes_reserved, info->bytes_may_use, - info->bytes_readonly); - spin_unlock(&info->lock); - - DUMP_BLOCK_RSV(fs_info, global_block_rsv); - DUMP_BLOCK_RSV(fs_info, trans_block_rsv); - DUMP_BLOCK_RSV(fs_info, chunk_block_rsv); - DUMP_BLOCK_RSV(fs_info, delayed_block_rsv); - DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv); - - if (!dump_block_groups) - return; - - down_read(&info->groups_sem); -again: - list_for_each_entry(cache, &info->block_groups[index], list) { - spin_lock(&cache->lock); - btrfs_info(fs_info, - "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s", - cache->key.objectid, cache->key.offset, - btrfs_block_group_used(&cache->item), cache->pinned, - cache->reserved, cache->ro ? "[readonly]" : ""); - btrfs_dump_free_space(cache, bytes); - spin_unlock(&cache->lock); - } - if (++index < BTRFS_NR_RAID_TYPES) - goto again; - up_read(&info->groups_sem); -} - /* * btrfs_reserve_extent - entry point to the extent allocator. Tries to find a * hole that is at least as big as @num_bytes. @@ -8113,12 +5934,13 @@ again: } else if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { struct btrfs_space_info *sinfo; - sinfo = __find_space_info(fs_info, flags); + sinfo = btrfs_find_space_info(fs_info, flags); btrfs_err(fs_info, "allocation failed flags %llu, wanted %llu", flags, num_bytes); if (sinfo) - dump_space_info(fs_info, sinfo, num_bytes, 1); + btrfs_dump_space_info(fs_info, sinfo, + num_bytes, 1); } } @@ -8456,73 +6278,6 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, return buf; } -static struct btrfs_block_rsv * -use_block_rsv(struct btrfs_trans_handle *trans, - struct btrfs_root *root, u32 blocksize) -{ - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_block_rsv *block_rsv; - struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; - int ret; - bool global_updated = false; - - block_rsv = get_block_rsv(trans, root); - - if (unlikely(block_rsv->size == 0)) - goto try_reserve; -again: - ret = block_rsv_use_bytes(block_rsv, blocksize); - if (!ret) - return block_rsv; - - if (block_rsv->failfast) - return ERR_PTR(ret); - - if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) { - global_updated = true; - update_global_block_rsv(fs_info); - goto again; - } - - /* - * The global reserve still exists to save us from ourselves, so don't - * warn_on if we are short on our delayed refs reserve. - */ - if (block_rsv->type != BTRFS_BLOCK_RSV_DELREFS && - btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { - static DEFINE_RATELIMIT_STATE(_rs, - DEFAULT_RATELIMIT_INTERVAL * 10, - /*DEFAULT_RATELIMIT_BURST*/ 1); - if (__ratelimit(&_rs)) - WARN(1, KERN_DEBUG - "BTRFS: block rsv returned %d\n", ret); - } -try_reserve: - ret = reserve_metadata_bytes(root, block_rsv, blocksize, - BTRFS_RESERVE_NO_FLUSH); - if (!ret) - return block_rsv; - /* - * If we couldn't reserve metadata bytes try and use some from - * the global reserve if its space type is the same as the global - * reservation. - */ - if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL && - block_rsv->space_info == global_rsv->space_info) { - ret = block_rsv_use_bytes(global_rsv, blocksize); - if (!ret) - return global_rsv; - } - return ERR_PTR(ret); -} - -static void unuse_block_rsv(struct btrfs_fs_info *fs_info, - struct btrfs_block_rsv *block_rsv, u32 blocksize) -{ - block_rsv_add_bytes(block_rsv, blocksize, false); - block_rsv_release_bytes(fs_info, block_rsv, NULL, 0, NULL); -} - /* * finds a free extent and does all the dirty work required for allocation * returns the tree buffer or an ERR_PTR on error. @@ -8555,7 +6310,7 @@ struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans, } #endif - block_rsv = use_block_rsv(trans, root, blocksize); + block_rsv = btrfs_use_block_rsv(trans, root, blocksize); if (IS_ERR(block_rsv)) return ERR_CAST(block_rsv); @@ -8613,7 +6368,7 @@ out_free_buf: out_free_reserved: btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 0); out_unuse: - unuse_block_rsv(fs_info, block_rsv, blocksize); + btrfs_unuse_block_rsv(fs_info, block_rsv, blocksize); return ERR_PTR(ret); } @@ -9552,9 +7307,8 @@ static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags) num_devices = fs_info->fs_devices->rw_devices; - stripped = BTRFS_BLOCK_GROUP_RAID0 | - BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 | - BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10; + stripped = BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID56_MASK | + BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10; if (num_devices == 1) { stripped |= BTRFS_BLOCK_GROUP_DUP; @@ -9565,7 +7319,7 @@ static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags) return stripped; /* turn mirroring into duplication */ - if (flags & (BTRFS_BLOCK_GROUP_RAID1 | + if (flags & (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)) return stripped | BTRFS_BLOCK_GROUP_DUP; } else { @@ -9636,7 +7390,7 @@ out: btrfs_info(cache->fs_info, "sinfo_used=%llu bg_num_bytes=%llu min_allocable=%llu", sinfo_used, num_bytes, min_allocable_bytes); - dump_space_info(cache->fs_info, cache->space_info, 0, 0); + btrfs_dump_space_info(cache->fs_info, cache->space_info, 0, 0); } return ret; } @@ -9678,8 +7432,7 @@ again: */ alloc_flags = update_block_group_flags(fs_info, cache->flags); if (alloc_flags != cache->flags) { - ret = do_chunk_alloc(trans, alloc_flags, - CHUNK_ALLOC_FORCE); + ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE); /* * ENOSPC is allowed here, we may have enough space * already allocated at the new raid level to @@ -9695,7 +7448,7 @@ again: if (!ret) goto out; alloc_flags = get_alloc_profile(fs_info, cache->space_info->flags); - ret = do_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE); + ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE); if (ret < 0) goto out; ret = inc_block_group_ro(cache, 0); @@ -9716,7 +7469,7 @@ int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type) { u64 alloc_flags = get_alloc_profile(trans->fs_info, type); - return do_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE); + return btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE); } /* @@ -9949,7 +7702,7 @@ static int find_first_block_group(struct btrfs_fs_info *fs_info, struct extent_map_tree *em_tree; struct extent_map *em; - em_tree = &root->fs_info->mapping_tree.map_tree; + em_tree = &root->fs_info->mapping_tree; read_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, found_key.objectid, found_key.offset); @@ -10102,7 +7855,7 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info) */ synchronize_rcu(); - release_global_block_rsv(info); + btrfs_release_global_block_rsv(info); while (!list_empty(&info->space_info)) { int i; @@ -10118,7 +7871,7 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info) if (WARN_ON(space_info->bytes_pinned > 0 || space_info->bytes_reserved > 0 || space_info->bytes_may_use > 0)) - dump_space_info(info, space_info, 0, 0); + btrfs_dump_space_info(info, space_info, 0, 0); list_del(&space_info->list); for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) { struct kobject *kobj; @@ -10141,7 +7894,6 @@ void btrfs_add_raid_kobjects(struct btrfs_fs_info *fs_info) struct btrfs_space_info *space_info; struct raid_kobject *rkobj; LIST_HEAD(list); - int index; int ret = 0; spin_lock(&fs_info->pending_raid_kobjs_lock); @@ -10149,11 +7901,10 @@ void btrfs_add_raid_kobjects(struct btrfs_fs_info *fs_info) spin_unlock(&fs_info->pending_raid_kobjs_lock); list_for_each_entry(rkobj, &list, list) { - space_info = __find_space_info(fs_info, rkobj->flags); - index = btrfs_bg_flags_to_raid_index(rkobj->flags); + space_info = btrfs_find_space_info(fs_info, rkobj->flags); ret = kobject_add(&rkobj->kobj, &space_info->kobj, - "%s", get_raid_name(index)); + "%s", btrfs_bg_type_to_raid_name(rkobj->flags)); if (ret) { kobject_put(&rkobj->kobj); break; @@ -10243,21 +7994,21 @@ btrfs_create_block_group_cache(struct btrfs_fs_info *fs_info, */ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) { - struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree; + struct extent_map_tree *map_tree = &fs_info->mapping_tree; struct extent_map *em; struct btrfs_block_group_cache *bg; u64 start = 0; int ret = 0; while (1) { - read_lock(&map_tree->map_tree.lock); + read_lock(&map_tree->lock); /* * lookup_extent_mapping will return the first extent map * intersecting the range, so setting @len to 1 is enough to * get the first chunk. */ - em = lookup_extent_mapping(&map_tree->map_tree, start, 1); - read_unlock(&map_tree->map_tree.lock); + em = lookup_extent_mapping(map_tree, start, 1); + read_unlock(&map_tree->lock); if (!em) break; @@ -10417,9 +8168,9 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) } trace_btrfs_add_block_group(info, cache, 0); - update_space_info(info, cache->flags, found_key.offset, - btrfs_block_group_used(&cache->item), - cache->bytes_super, &space_info); + btrfs_update_space_info(info, cache->flags, found_key.offset, + btrfs_block_group_used(&cache->item), + cache->bytes_super, &space_info); cache->space_info = space_info; @@ -10437,9 +8188,8 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) list_for_each_entry_rcu(space_info, &info->space_info, list) { if (!(get_alloc_profile(info, space_info->flags) & (BTRFS_BLOCK_GROUP_RAID10 | - BTRFS_BLOCK_GROUP_RAID1 | - BTRFS_BLOCK_GROUP_RAID5 | - BTRFS_BLOCK_GROUP_RAID6 | + BTRFS_BLOCK_GROUP_RAID1_MASK | + BTRFS_BLOCK_GROUP_RAID56_MASK | BTRFS_BLOCK_GROUP_DUP))) continue; /* @@ -10457,7 +8207,7 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) } btrfs_add_raid_kobjects(info); - init_global_block_rsv(info); + btrfs_init_global_block_rsv(info); ret = check_chunk_block_group_mappings(info); error: btrfs_free_path(path); @@ -10554,7 +8304,7 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, * assigned to our block group. We want our bg to be added to the rbtree * with its ->space_info set. */ - cache->space_info = __find_space_info(fs_info, cache->flags); + cache->space_info = btrfs_find_space_info(fs_info, cache->flags); ASSERT(cache->space_info); ret = btrfs_add_block_group_cache(fs_info, cache); @@ -10569,9 +8319,9 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, * the rbtree, update the space info's counters. */ trace_btrfs_add_block_group(fs_info, cache, 1); - update_space_info(fs_info, cache->flags, size, bytes_used, + btrfs_update_space_info(fs_info, cache->flags, size, bytes_used, cache->bytes_super, &cache->space_info); - update_global_block_rsv(fs_info); + btrfs_update_global_block_rsv(fs_info); link_block_group(cache); @@ -10598,6 +8348,35 @@ static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) write_sequnlock(&fs_info->profiles_lock); } +/* + * Clear incompat bits for the following feature(s): + * + * - RAID56 - in case there's neither RAID5 nor RAID6 profile block group + * in the whole filesystem + */ +static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags) +{ + if (flags & BTRFS_BLOCK_GROUP_RAID56_MASK) { + struct list_head *head = &fs_info->space_info; + struct btrfs_space_info *sinfo; + + list_for_each_entry_rcu(sinfo, head, list) { + bool found = false; + + down_read(&sinfo->groups_sem); + if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID5])) + found = true; + if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID6])) + found = true; + up_read(&sinfo->groups_sem); + + if (found) + return; + } + btrfs_clear_fs_incompat(fs_info, RAID56); + } +} + int btrfs_remove_block_group(struct btrfs_trans_handle *trans, u64 group_start, struct extent_map *em) { @@ -10744,6 +8523,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, clear_avail_alloc_bits(fs_info, block_group->flags); } up_write(&block_group->space_info->groups_sem); + clear_incompat_bg_bits(fs_info, block_group->flags); if (kobj) { kobject_del(kobj); kobject_put(kobj); @@ -10853,7 +8633,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, if (remove_em) { struct extent_map_tree *em_tree; - em_tree = &fs_info->mapping_tree.map_tree; + em_tree = &fs_info->mapping_tree; write_lock(&em_tree->lock); remove_extent_mapping(em_tree, em); write_unlock(&em_tree->lock); @@ -10871,7 +8651,7 @@ struct btrfs_trans_handle * btrfs_start_trans_remove_block_group(struct btrfs_fs_info *fs_info, const u64 chunk_offset) { - struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree; + struct extent_map_tree *em_tree = &fs_info->mapping_tree; struct extent_map *em; struct map_lookup *map; unsigned int num_items; @@ -11020,7 +8800,8 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) spin_lock(&space_info->lock); spin_lock(&block_group->lock); - update_bytes_pinned(space_info, -block_group->pinned); + btrfs_space_info_update_bytes_pinned(fs_info, space_info, + -block_group->pinned); space_info->bytes_readonly += block_group->pinned; percpu_counter_add_batch(&space_info->total_bytes_pinned, -block_group->pinned, @@ -11076,43 +8857,6 @@ next: spin_unlock(&fs_info->unused_bgs_lock); } -int btrfs_init_space_info(struct btrfs_fs_info *fs_info) -{ - struct btrfs_super_block *disk_super; - u64 features; - u64 flags; - int mixed = 0; - int ret; - - disk_super = fs_info->super_copy; - if (!btrfs_super_root(disk_super)) - return -EINVAL; - - features = btrfs_super_incompat_flags(disk_super); - if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) - mixed = 1; - - flags = BTRFS_BLOCK_GROUP_SYSTEM; - ret = create_space_info(fs_info, flags); - if (ret) - goto out; - - if (mixed) { - flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA; - ret = create_space_info(fs_info, flags); - } else { - flags = BTRFS_BLOCK_GROUP_METADATA; - ret = create_space_info(fs_info, flags); - if (ret) - goto out; - - flags = BTRFS_BLOCK_GROUP_DATA; - ret = create_space_info(fs_info, flags); - } -out: - return ret; -} - int btrfs_error_unpin_extent_range(struct btrfs_fs_info *fs_info, u64 start, u64 end) { @@ -11171,12 +8915,17 @@ static int btrfs_trim_free_extents(struct btrfs_device *device, u64 *trimmed) find_first_clear_extent_bit(&device->alloc_state, start, &start, &end, CHUNK_TRIMMED | CHUNK_ALLOCATED); + + /* Ensure we skip the reserved area in the first 1M */ + start = max_t(u64, start, SZ_1M); + /* * If find_first_clear_extent_bit find a range that spans the * end of the device it will set end to -1, in this case it's up * to the caller to trim the value to the size of the device. */ end = min(end, device->total_bytes - 1); + len = end - start + 1; /* We didn't find any extents */ diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 5106008f5e28..1ff438fd5bc2 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -359,6 +359,24 @@ do_insert: return NULL; } +/** + * __etree_search - searche @tree for an entry that contains @offset. Such + * entry would have entry->start <= offset && entry->end >= offset. + * + * @tree - the tree to search + * @offset - offset that should fall within an entry in @tree + * @next_ret - pointer to the first entry whose range ends after @offset + * @prev - pointer to the first entry whose range begins before @offset + * @p_ret - pointer where new node should be anchored (used when inserting an + * entry in the tree) + * @parent_ret - points to entry which would have been the parent of the entry, + * containing @offset + * + * This function returns a pointer to the entry that contains @offset byte + * address. If no such entry exists, then NULL is returned and the other + * pointer arguments to the function are filled, otherwise the found entry is + * returned and other pointers are left untouched. + */ static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset, struct rb_node **next_ret, struct rb_node **prev_ret, @@ -504,9 +522,11 @@ static int insert_state(struct extent_io_tree *tree, { struct rb_node *node; - if (end < start) - WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n", - end, start); + if (end < start) { + btrfs_err(tree->fs_info, + "insert state: end < start %llu %llu", end, start); + WARN_ON(1); + } state->start = start; state->end = end; @@ -516,7 +536,8 @@ static int insert_state(struct extent_io_tree *tree, if (node) { struct extent_state *found; found = rb_entry(node, struct extent_state, rb_node); - pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n", + btrfs_err(tree->fs_info, + "found node %llu %llu on insert of %llu %llu", found->start, found->end, start, end); return -EEXIST; } @@ -1537,8 +1558,8 @@ out: } /** - * find_first_clear_extent_bit - finds the first range that has @bits not set - * and that starts after @start + * find_first_clear_extent_bit - find the first range that has @bits not set. + * This range could start before @start. * * @tree - the tree to search * @start - the offset at/after which the found extent should start @@ -1578,12 +1599,52 @@ void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, goto out; } } + /* + * At this point 'node' either contains 'start' or start is + * before 'node' + */ state = rb_entry(node, struct extent_state, rb_node); - if (in_range(start, state->start, state->end - state->start + 1) && - (state->state & bits)) { - start = state->end + 1; + + if (in_range(start, state->start, state->end - state->start + 1)) { + if (state->state & bits) { + /* + * |--range with bits sets--| + * | + * start + */ + start = state->end + 1; + } else { + /* + * 'start' falls within a range that doesn't + * have the bits set, so take its start as + * the beginning of the desired range + * + * |--range with bits cleared----| + * | + * start + */ + *start_ret = state->start; + break; + } } else { - *start_ret = start; + /* + * |---prev range---|---hole/unset---|---node range---| + * | + * start + * + * or + * + * |---hole/unset--||--first node--| + * 0 | + * start + */ + if (prev) { + state = rb_entry(prev, struct extent_state, + rb_node); + *start_ret = state->end + 1; + } else { + *start_ret = 0; + } break; } } @@ -1719,10 +1780,10 @@ static noinline int lock_delalloc_pages(struct inode *inode, */ EXPORT_FOR_TESTS noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, - struct extent_io_tree *tree, struct page *locked_page, u64 *start, u64 *end) { + struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; u64 max_bytes = BTRFS_MAX_EXTENT_SIZE; u64 delalloc_start; u64 delalloc_end; @@ -2800,12 +2861,11 @@ static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio) * never fail. We're returning a bio right now but you can call btrfs_io_bio * for the appropriate container_of magic */ -struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte) +struct bio *btrfs_bio_alloc(u64 first_byte) { struct bio *bio; bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset); - bio_set_dev(bio, bdev); bio->bi_iter.bi_sector = first_byte >> 9; btrfs_io_bio_init(btrfs_io_bio(bio)); return bio; @@ -2916,7 +2976,8 @@ static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree, } } - bio = btrfs_bio_alloc(bdev, offset); + bio = btrfs_bio_alloc(offset); + bio_set_dev(bio, bdev); bio_add_page(bio, page, page_size, pg_offset); bio->bi_end_io = end_io_func; bio->bi_private = tree; @@ -3204,21 +3265,10 @@ static inline void contiguous_readpages(struct extent_io_tree *tree, unsigned long *bio_flags, u64 *prev_em_start) { - struct inode *inode; - struct btrfs_ordered_extent *ordered; + struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host); int index; - inode = pages[0]->mapping->host; - while (1) { - lock_extent(tree, start, end); - ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start, - end - start + 1); - if (!ordered) - break; - unlock_extent(tree, start, end); - btrfs_start_ordered_extent(inode, ordered, 1); - btrfs_put_ordered_extent(ordered); - } + btrfs_lock_and_flush_ordered_range(tree, inode, start, end, NULL); for (index = 0; index < nr_pages; index++) { __do_readpage(tree, pages[index], btrfs_get_extent, em_cached, @@ -3234,22 +3284,12 @@ static int __extent_read_full_page(struct extent_io_tree *tree, unsigned long *bio_flags, unsigned int read_flags) { - struct inode *inode = page->mapping->host; - struct btrfs_ordered_extent *ordered; + struct btrfs_inode *inode = BTRFS_I(page->mapping->host); u64 start = page_offset(page); u64 end = start + PAGE_SIZE - 1; int ret; - while (1) { - lock_extent(tree, start, end); - ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start, - PAGE_SIZE); - if (!ordered) - break; - unlock_extent(tree, start, end); - btrfs_start_ordered_extent(inode, ordered, 1); - btrfs_put_ordered_extent(ordered); - } + btrfs_lock_and_flush_ordered_range(tree, inode, start, end, NULL); ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num, bio_flags, read_flags, NULL); @@ -3290,7 +3330,6 @@ static noinline_for_stack int writepage_delalloc(struct inode *inode, struct page *page, struct writeback_control *wbc, u64 delalloc_start, unsigned long *nr_written) { - struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; u64 page_end = delalloc_start + PAGE_SIZE - 1; bool found; u64 delalloc_to_write = 0; @@ -3300,8 +3339,7 @@ static noinline_for_stack int writepage_delalloc(struct inode *inode, while (delalloc_end < page_end) { - found = find_lock_delalloc_range(inode, tree, - page, + found = find_lock_delalloc_range(inode, page, &delalloc_start, &delalloc_end); if (!found) { @@ -3310,7 +3348,6 @@ static noinline_for_stack int writepage_delalloc(struct inode *inode, } ret = btrfs_run_delalloc_range(inode, page, delalloc_start, delalloc_end, &page_started, nr_written, wbc); - /* File system has been set read-only */ if (ret) { SetPageError(page); /* @@ -4542,6 +4579,8 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, struct btrfs_path *path; struct btrfs_root *root = BTRFS_I(inode)->root; struct fiemap_cache cache = { 0 }; + struct ulist *roots; + struct ulist *tmp_ulist; int end = 0; u64 em_start = 0; u64 em_len = 0; @@ -4555,6 +4594,13 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, return -ENOMEM; path->leave_spinning = 1; + roots = ulist_alloc(GFP_KERNEL); + tmp_ulist = ulist_alloc(GFP_KERNEL); + if (!roots || !tmp_ulist) { + ret = -ENOMEM; + goto out_free_ulist; + } + start = round_down(start, btrfs_inode_sectorsize(inode)); len = round_up(max, btrfs_inode_sectorsize(inode)) - start; @@ -4565,8 +4611,7 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(BTRFS_I(inode)), -1, 0); if (ret < 0) { - btrfs_free_path(path); - return ret; + goto out_free_ulist; } else { WARN_ON(!ret); if (ret == 1) @@ -4675,7 +4720,7 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, */ ret = btrfs_check_shared(root, btrfs_ino(BTRFS_I(inode)), - bytenr); + bytenr, roots, tmp_ulist); if (ret < 0) goto out_free; if (ret) @@ -4718,9 +4763,13 @@ out_free: ret = emit_last_fiemap_cache(fieinfo, &cache); free_extent_map(em); out: - btrfs_free_path(path); unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1, &cached_state); + +out_free_ulist: + btrfs_free_path(path); + ulist_free(roots); + ulist_free(tmp_ulist); return ret; } @@ -4808,7 +4857,7 @@ __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, eb->bflags = 0; rwlock_init(&eb->lock); atomic_set(&eb->blocking_readers, 0); - atomic_set(&eb->blocking_writers, 0); + eb->blocking_writers = 0; eb->lock_nested = false; init_waitqueue_head(&eb->write_lock_wq); init_waitqueue_head(&eb->read_lock_wq); @@ -4827,10 +4876,10 @@ __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE); #ifdef CONFIG_BTRFS_DEBUG - atomic_set(&eb->spinning_writers, 0); + eb->spinning_writers = 0; atomic_set(&eb->spinning_readers, 0); atomic_set(&eb->read_locks, 0); - atomic_set(&eb->write_locks, 0); + eb->write_locks = 0; #endif return eb; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index aa18a16a6ed7..401423b16976 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -167,7 +167,7 @@ struct extent_buffer { struct rcu_head rcu_head; pid_t lock_owner; - atomic_t blocking_writers; + int blocking_writers; atomic_t blocking_readers; bool lock_nested; /* >= 0 if eb belongs to a log tree, -1 otherwise */ @@ -187,10 +187,10 @@ struct extent_buffer { wait_queue_head_t read_lock_wq; struct page *pages[INLINE_EXTENT_BUFFER_PAGES]; #ifdef CONFIG_BTRFS_DEBUG - atomic_t spinning_writers; + int spinning_writers; atomic_t spinning_readers; atomic_t read_locks; - atomic_t write_locks; + int write_locks; struct list_head leak_list; #endif }; @@ -497,7 +497,7 @@ void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end, u64 delalloc_end, struct page *locked_page, unsigned bits_to_clear, unsigned long page_ops); -struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte); +struct bio *btrfs_bio_alloc(u64 first_byte); struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs); struct bio *btrfs_bio_clone(struct bio *bio); struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size); @@ -549,7 +549,7 @@ int free_io_failure(struct extent_io_tree *failure_tree, struct extent_io_tree *io_tree, struct io_failure_record *rec); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS -bool find_lock_delalloc_range(struct inode *inode, struct extent_io_tree *tree, +bool find_lock_delalloc_range(struct inode *inode, struct page *locked_page, u64 *start, u64 *end); #endif diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index d431ea8198e4..1a599f50837b 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -8,6 +8,7 @@ #include <linux/pagemap.h> #include <linux/highmem.h> #include <linux/sched/mm.h> +#include <crypto/hash.h> #include "ctree.h" #include "disk-io.h" #include "transaction.h" @@ -22,9 +23,13 @@ #define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \ PAGE_SIZE)) -#define MAX_ORDERED_SUM_BYTES(fs_info) ((PAGE_SIZE - \ - sizeof(struct btrfs_ordered_sum)) / \ - sizeof(u32) * (fs_info)->sectorsize) +static inline u32 max_ordered_sum_bytes(struct btrfs_fs_info *fs_info, + u16 csum_size) +{ + u32 ncsums = (PAGE_SIZE - sizeof(struct btrfs_ordered_sum)) / csum_size; + + return ncsums * fs_info->sectorsize; +} int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root, @@ -144,7 +149,7 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, } static blk_status_t __btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, - u64 logical_offset, u32 *dst, int dio) + u64 logical_offset, u8 *dst, int dio) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct bio_vec bvec; @@ -182,7 +187,7 @@ static blk_status_t __btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio } csum = btrfs_bio->csum; } else { - csum = (u8 *)dst; + csum = dst; } if (bio->bi_iter.bi_size > PAGE_SIZE * 8) @@ -211,7 +216,7 @@ static blk_status_t __btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio if (!dio) offset = page_offset(bvec.bv_page) + bvec.bv_offset; count = btrfs_find_ordered_sum(inode, offset, disk_bytenr, - (u32 *)csum, nblocks); + csum, nblocks); if (count) goto found; @@ -283,7 +288,8 @@ next: return 0; } -blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u32 *dst) +blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, + u8 *dst) { return __btrfs_lookup_bio_sums(inode, bio, 0, dst, 0); } @@ -374,7 +380,7 @@ int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, struct btrfs_csum_item); while (start < csum_end) { size = min_t(size_t, csum_end - start, - MAX_ORDERED_SUM_BYTES(fs_info)); + max_ordered_sum_bytes(fs_info, csum_size)); sums = kzalloc(btrfs_ordered_sum_size(fs_info, size), GFP_NOFS); if (!sums) { @@ -427,6 +433,7 @@ blk_status_t btrfs_csum_one_bio(struct inode *inode, struct bio *bio, u64 file_start, int contig) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); struct btrfs_ordered_sum *sums; struct btrfs_ordered_extent *ordered = NULL; char *data; @@ -439,6 +446,7 @@ blk_status_t btrfs_csum_one_bio(struct inode *inode, struct bio *bio, int i; u64 offset; unsigned nofs_flag; + const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); nofs_flag = memalloc_nofs_save(); sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size), @@ -459,6 +467,8 @@ blk_status_t btrfs_csum_one_bio(struct inode *inode, struct bio *bio, sums->bytenr = (u64)bio->bi_iter.bi_sector << 9; index = 0; + shash->tfm = fs_info->csum_shash; + bio_for_each_segment(bvec, bio, iter) { if (!contig) offset = page_offset(bvec.bv_page) + bvec.bv_offset; @@ -498,17 +508,14 @@ blk_status_t btrfs_csum_one_bio(struct inode *inode, struct bio *bio, index = 0; } - sums->sums[index] = ~(u32)0; + crypto_shash_init(shash); data = kmap_atomic(bvec.bv_page); - sums->sums[index] - = btrfs_csum_data(data + bvec.bv_offset - + (i * fs_info->sectorsize), - sums->sums[index], - fs_info->sectorsize); + crypto_shash_update(shash, data + bvec.bv_offset + + (i * fs_info->sectorsize), + fs_info->sectorsize); kunmap_atomic(data); - btrfs_csum_final(sums->sums[index], - (char *)(sums->sums + index)); - index++; + crypto_shash_final(shash, (char *)(sums->sums + index)); + index += csum_size; offset += fs_info->sectorsize; this_sum_bytes += fs_info->sectorsize; total_bytes += fs_info->sectorsize; @@ -904,9 +911,9 @@ found: write_extent_buffer(leaf, sums->sums + index, (unsigned long)item, ins_size); + index += ins_size; ins_size /= csum_size; total_bytes += ins_size * fs_info->sectorsize; - index += ins_size; btrfs_mark_buffer_dirty(path->nodes[0]); if (total_bytes < sums->len) { diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 89f5be2bfb43..58a18ed11546 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -26,6 +26,7 @@ #include "volumes.h" #include "qgroup.h" #include "compression.h" +#include "delalloc-space.h" static struct kmem_cache *btrfs_inode_defrag_cachep; /* @@ -1550,30 +1551,20 @@ static noinline int check_can_nocow(struct btrfs_inode *inode, loff_t pos, { struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_root *root = inode->root; - struct btrfs_ordered_extent *ordered; u64 lockstart, lockend; u64 num_bytes; int ret; ret = btrfs_start_write_no_snapshotting(root); if (!ret) - return -ENOSPC; + return -EAGAIN; lockstart = round_down(pos, fs_info->sectorsize); lockend = round_up(pos + *write_bytes, fs_info->sectorsize) - 1; - while (1) { - lock_extent(&inode->io_tree, lockstart, lockend); - ordered = btrfs_lookup_ordered_range(inode, lockstart, - lockend - lockstart + 1); - if (!ordered) { - break; - } - unlock_extent(&inode->io_tree, lockstart, lockend); - btrfs_start_ordered_extent(&inode->vfs_inode, ordered, 1); - btrfs_put_ordered_extent(ordered); - } + btrfs_lock_and_flush_ordered_range(&inode->io_tree, inode, lockstart, + lockend, NULL); num_bytes = lockend - lockstart + 1; ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes, @@ -2721,6 +2712,11 @@ out_only_mutex: * for detecting, at fsync time, if the inode isn't yet in the * log tree or it's there but not up to date. */ + struct timespec64 now = current_time(inode); + + inode_inc_iversion(inode); + inode->i_mtime = now; + inode->i_ctime = now; trans = btrfs_start_transaction(root, 1); if (IS_ERR(trans)) { err = PTR_ERR(trans); @@ -2801,9 +2797,9 @@ static int btrfs_fallocate_update_isize(struct inode *inode, } enum { - RANGE_BOUNDARY_WRITTEN_EXTENT = 0, - RANGE_BOUNDARY_PREALLOC_EXTENT = 1, - RANGE_BOUNDARY_HOLE = 2, + RANGE_BOUNDARY_WRITTEN_EXTENT, + RANGE_BOUNDARY_PREALLOC_EXTENT, + RANGE_BOUNDARY_HOLE, }; static int btrfs_zero_range_check_range_boundary(struct inode *inode, diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index f74dc259307b..062be9dde4c6 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -18,6 +18,8 @@ #include "extent_io.h" #include "inode-map.h" #include "volumes.h" +#include "space-info.h" +#include "delalloc-space.h" #define BITS_PER_BITMAP (PAGE_SIZE * 8UL) #define MAX_CACHE_BYTES_PER_GIG SZ_32K @@ -465,9 +467,8 @@ static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index) if (index == 0) offset = sizeof(u32) * io_ctl->num_pages; - crc = btrfs_csum_data(io_ctl->orig + offset, crc, - PAGE_SIZE - offset); - btrfs_csum_final(crc, (u8 *)&crc); + crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); + btrfs_crc32c_final(crc, (u8 *)&crc); io_ctl_unmap_page(io_ctl); tmp = page_address(io_ctl->pages[0]); tmp += index; @@ -493,9 +494,8 @@ static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index) val = *tmp; io_ctl_map_page(io_ctl, 0); - crc = btrfs_csum_data(io_ctl->orig + offset, crc, - PAGE_SIZE - offset); - btrfs_csum_final(crc, (u8 *)&crc); + crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); + btrfs_crc32c_final(crc, (u8 *)&crc); if (val != crc) { btrfs_err_rl(io_ctl->fs_info, "csum mismatch on free space cache"); @@ -3166,8 +3166,8 @@ static int do_trimming(struct btrfs_block_group_cache *block_group, space_info->bytes_readonly += reserved_bytes; block_group->reserved -= reserved_bytes; space_info->bytes_reserved -= reserved_bytes; - spin_unlock(&space_info->lock); spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); } return ret; @@ -3358,7 +3358,7 @@ void btrfs_put_block_group_trimming(struct btrfs_block_group_cache *block_group) if (cleanup) { mutex_lock(&fs_info->chunk_mutex); - em_tree = &fs_info->mapping_tree.map_tree; + em_tree = &fs_info->mapping_tree; write_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, block_group->key.objectid, 1); diff --git a/fs/btrfs/inode-map.c b/fs/btrfs/inode-map.c index ffca2abf13d0..2e8bb402050b 100644 --- a/fs/btrfs/inode-map.c +++ b/fs/btrfs/inode-map.c @@ -11,6 +11,7 @@ #include "free-space-cache.h" #include "inode-map.h" #include "transaction.h" +#include "delalloc-space.h" static int caching_kthread(void *data) { diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index a2aabdb85226..1af069a9a0c7 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -47,6 +47,7 @@ #include "props.h" #include "qgroup.h" #include "dedupe.h" +#include "delalloc-space.h" struct btrfs_iget_args { struct btrfs_key *location; @@ -1932,17 +1933,19 @@ int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, u64 length = 0; u64 map_length; int ret; + struct btrfs_io_geometry geom; if (bio_flags & EXTENT_BIO_COMPRESSED) return 0; length = bio->bi_iter.bi_size; map_length = length; - ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, - NULL, 0); + ret = btrfs_get_io_geometry(fs_info, btrfs_op(bio), logical, map_length, + &geom); if (ret < 0) return ret; - if (map_length < length + size) + + if (geom.len < length + size) return 1; return 0; } @@ -3203,16 +3206,23 @@ static int __readpage_endio_check(struct inode *inode, int icsum, struct page *page, int pgoff, u64 start, size_t len) { + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); char *kaddr; - u32 csum_expected; - u32 csum = ~(u32)0; + u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + u8 *csum_expected; + u8 csum[BTRFS_CSUM_SIZE]; - csum_expected = *(((u32 *)io_bio->csum) + icsum); + csum_expected = ((u8 *)io_bio->csum) + icsum * csum_size; kaddr = kmap_atomic(page); - csum = btrfs_csum_data(kaddr + pgoff, csum, len); - btrfs_csum_final(csum, (u8 *)&csum); - if (csum != csum_expected) + shash->tfm = fs_info->csum_shash; + + crypto_shash_init(shash); + crypto_shash_update(shash, kaddr + pgoff, len); + crypto_shash_final(shash, csum); + + if (memcmp(csum, csum_expected, csum_size)) goto zeroit; kunmap_atomic(kaddr); @@ -3286,6 +3296,28 @@ void btrfs_add_delayed_iput(struct inode *inode) wake_up_process(fs_info->cleaner_kthread); } +static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode) +{ + list_del_init(&inode->delayed_iput); + spin_unlock(&fs_info->delayed_iput_lock); + iput(&inode->vfs_inode); + if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) + wake_up(&fs_info->delayed_iputs_wait); + spin_lock(&fs_info->delayed_iput_lock); +} + +static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode) +{ + if (!list_empty(&inode->delayed_iput)) { + spin_lock(&fs_info->delayed_iput_lock); + if (!list_empty(&inode->delayed_iput)) + run_delayed_iput_locked(fs_info, inode); + spin_unlock(&fs_info->delayed_iput_lock); + } +} + void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) { @@ -3295,12 +3327,7 @@ void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) inode = list_first_entry(&fs_info->delayed_iputs, struct btrfs_inode, delayed_iput); - list_del_init(&inode->delayed_iput); - spin_unlock(&fs_info->delayed_iput_lock); - iput(&inode->vfs_inode); - if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) - wake_up(&fs_info->delayed_iputs_wait); - spin_lock(&fs_info->delayed_iput_lock); + run_delayed_iput_locked(fs_info, inode); } spin_unlock(&fs_info->delayed_iput_lock); } @@ -3935,9 +3962,7 @@ static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_path *path; int ret = 0; - struct extent_buffer *leaf; struct btrfs_dir_item *di; - struct btrfs_key key; u64 index; u64 ino = btrfs_ino(inode); u64 dir_ino = btrfs_ino(dir); @@ -3955,8 +3980,6 @@ static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, ret = di ? PTR_ERR(di) : -ENOENT; goto err; } - leaf = path->nodes[0]; - btrfs_dir_item_key_to_cpu(leaf, di, &key); ret = btrfs_delete_one_dir_name(trans, root, path, di); if (ret) goto err; @@ -4009,6 +4032,17 @@ skip_backref: ret = 0; else if (ret) btrfs_abort_transaction(trans, ret); + + /* + * If we have a pending delayed iput we could end up with the final iput + * being run in btrfs-cleaner context. If we have enough of these built + * up we can end up burning a lot of time in btrfs-cleaner without any + * way to throttle the unlinks. Since we're currently holding a ref on + * the inode we can run the delayed iput here without any issues as the + * final iput won't be done until after we drop the ref we're currently + * holding. + */ + btrfs_run_delayed_iput(fs_info, inode); err: btrfs_free_path(path); if (ret) @@ -5008,21 +5042,8 @@ int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size) if (size <= hole_start) return 0; - while (1) { - struct btrfs_ordered_extent *ordered; - - lock_extent_bits(io_tree, hole_start, block_end - 1, - &cached_state); - ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start, - block_end - hole_start); - if (!ordered) - break; - unlock_extent_cached(io_tree, hole_start, block_end - 1, - &cached_state); - btrfs_start_ordered_extent(inode, ordered, 1); - btrfs_put_ordered_extent(ordered); - } - + btrfs_lock_and_flush_ordered_range(io_tree, BTRFS_I(inode), hole_start, + block_end - 1, &cached_state); cur_offset = hole_start; while (1) { em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset, @@ -8318,22 +8339,21 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) struct bio *orig_bio = dip->orig_bio; u64 start_sector = orig_bio->bi_iter.bi_sector; u64 file_offset = dip->logical_offset; - u64 map_length; int async_submit = 0; u64 submit_len; int clone_offset = 0; int clone_len; int ret; blk_status_t status; + struct btrfs_io_geometry geom; - map_length = orig_bio->bi_iter.bi_size; - submit_len = map_length; - ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9, - &map_length, NULL, 0); + submit_len = orig_bio->bi_iter.bi_size; + ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio), + start_sector << 9, submit_len, &geom); if (ret) return -EIO; - if (map_length >= submit_len) { + if (geom.len >= submit_len) { bio = orig_bio; dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED; goto submit; @@ -8346,10 +8366,10 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) async_submit = 1; /* bio split */ - ASSERT(map_length <= INT_MAX); + ASSERT(geom.len <= INT_MAX); atomic_inc(&dip->pending_bios); do { - clone_len = min_t(int, submit_len, map_length); + clone_len = min_t(int, submit_len, geom.len); /* * This will never fail as it's passing GPF_NOFS and @@ -8386,9 +8406,8 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) start_sector += clone_len >> 9; file_offset += clone_len; - map_length = submit_len; - ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), - start_sector << 9, &map_length, NULL, 0); + ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio), + start_sector << 9, submit_len, &geom); if (ret) goto out_err; } while (submit_len > 0); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index cfeff1b8dce0..818f7ec8bb0e 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -43,6 +43,8 @@ #include "qgroup.h" #include "tree-log.h" #include "compression.h" +#include "space-info.h" +#include "delalloc-space.h" #ifdef CONFIG_64BIT /* If we have a 32-bit userspace and 64-bit kernel, then the UAPI @@ -3993,6 +3995,27 @@ static int btrfs_remap_file_range_prep(struct file *file_in, loff_t pos_in, if (!same_inode) inode_dio_wait(inode_out); + /* + * Workaround to make sure NOCOW buffered write reach disk as NOCOW. + * + * Btrfs' back references do not have a block level granularity, they + * work at the whole extent level. + * NOCOW buffered write without data space reserved may not be able + * to fall back to CoW due to lack of data space, thus could cause + * data loss. + * + * Here we take a shortcut by flushing the whole inode, so that all + * nocow write should reach disk as nocow before we increase the + * reference of the extent. We could do better by only flushing NOCOW + * data, but that needs extra accounting. + * + * Also we don't need to check ASYNC_EXTENT, as async extent will be + * CoWed anyway, not affecting nocow part. + */ + ret = filemap_flush(inode_in->i_mapping); + if (ret < 0) + return ret; + ret = btrfs_wait_ordered_range(inode_in, ALIGN_DOWN(pos_in, bs), wb_len); if (ret < 0) diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c index 2f6c3c7851ed..98fccce4208c 100644 --- a/fs/btrfs/locking.c +++ b/fs/btrfs/locking.c @@ -15,19 +15,19 @@ #ifdef CONFIG_BTRFS_DEBUG static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { - WARN_ON(atomic_read(&eb->spinning_writers)); - atomic_inc(&eb->spinning_writers); + WARN_ON(eb->spinning_writers); + eb->spinning_writers++; } static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { - WARN_ON(atomic_read(&eb->spinning_writers) != 1); - atomic_dec(&eb->spinning_writers); + WARN_ON(eb->spinning_writers != 1); + eb->spinning_writers--; } static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { - WARN_ON(atomic_read(&eb->spinning_writers)); + WARN_ON(eb->spinning_writers); } static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) @@ -58,17 +58,17 @@ static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { - atomic_inc(&eb->write_locks); + eb->write_locks++; } static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { - atomic_dec(&eb->write_locks); + eb->write_locks--; } void btrfs_assert_tree_locked(struct extent_buffer *eb) { - BUG_ON(!atomic_read(&eb->write_locks)); + BUG_ON(!eb->write_locks); } #else @@ -111,10 +111,10 @@ void btrfs_set_lock_blocking_write(struct extent_buffer *eb) */ if (eb->lock_nested && current->pid == eb->lock_owner) return; - if (atomic_read(&eb->blocking_writers) == 0) { + if (eb->blocking_writers == 0) { btrfs_assert_spinning_writers_put(eb); btrfs_assert_tree_locked(eb); - atomic_inc(&eb->blocking_writers); + eb->blocking_writers++; write_unlock(&eb->lock); } } @@ -148,12 +148,11 @@ void btrfs_clear_lock_blocking_write(struct extent_buffer *eb) */ if (eb->lock_nested && current->pid == eb->lock_owner) return; - BUG_ON(atomic_read(&eb->blocking_writers) != 1); write_lock(&eb->lock); + BUG_ON(eb->blocking_writers != 1); btrfs_assert_spinning_writers_get(eb); - /* atomic_dec_and_test implies a barrier */ - if (atomic_dec_and_test(&eb->blocking_writers)) - cond_wake_up_nomb(&eb->write_lock_wq); + if (--eb->blocking_writers == 0) + cond_wake_up(&eb->write_lock_wq); } /* @@ -167,12 +166,10 @@ void btrfs_tree_read_lock(struct extent_buffer *eb) if (trace_btrfs_tree_read_lock_enabled()) start_ns = ktime_get_ns(); again: - BUG_ON(!atomic_read(&eb->blocking_writers) && - current->pid == eb->lock_owner); - read_lock(&eb->lock); - if (atomic_read(&eb->blocking_writers) && - current->pid == eb->lock_owner) { + BUG_ON(eb->blocking_writers == 0 && + current->pid == eb->lock_owner); + if (eb->blocking_writers && current->pid == eb->lock_owner) { /* * This extent is already write-locked by our thread. We allow * an additional read lock to be added because it's for the same @@ -185,10 +182,10 @@ again: trace_btrfs_tree_read_lock(eb, start_ns); return; } - if (atomic_read(&eb->blocking_writers)) { + if (eb->blocking_writers) { read_unlock(&eb->lock); wait_event(eb->write_lock_wq, - atomic_read(&eb->blocking_writers) == 0); + eb->blocking_writers == 0); goto again; } btrfs_assert_tree_read_locks_get(eb); @@ -203,11 +200,11 @@ again: */ int btrfs_tree_read_lock_atomic(struct extent_buffer *eb) { - if (atomic_read(&eb->blocking_writers)) + if (eb->blocking_writers) return 0; read_lock(&eb->lock); - if (atomic_read(&eb->blocking_writers)) { + if (eb->blocking_writers) { read_unlock(&eb->lock); return 0; } @@ -223,13 +220,13 @@ int btrfs_tree_read_lock_atomic(struct extent_buffer *eb) */ int btrfs_try_tree_read_lock(struct extent_buffer *eb) { - if (atomic_read(&eb->blocking_writers)) + if (eb->blocking_writers) return 0; if (!read_trylock(&eb->lock)) return 0; - if (atomic_read(&eb->blocking_writers)) { + if (eb->blocking_writers) { read_unlock(&eb->lock); return 0; } @@ -245,13 +242,11 @@ int btrfs_try_tree_read_lock(struct extent_buffer *eb) */ int btrfs_try_tree_write_lock(struct extent_buffer *eb) { - if (atomic_read(&eb->blocking_writers) || - atomic_read(&eb->blocking_readers)) + if (eb->blocking_writers || atomic_read(&eb->blocking_readers)) return 0; write_lock(&eb->lock); - if (atomic_read(&eb->blocking_writers) || - atomic_read(&eb->blocking_readers)) { + if (eb->blocking_writers || atomic_read(&eb->blocking_readers)) { write_unlock(&eb->lock); return 0; } @@ -322,10 +317,9 @@ void btrfs_tree_lock(struct extent_buffer *eb) WARN_ON(eb->lock_owner == current->pid); again: wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0); - wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0); + wait_event(eb->write_lock_wq, eb->blocking_writers == 0); write_lock(&eb->lock); - if (atomic_read(&eb->blocking_readers) || - atomic_read(&eb->blocking_writers)) { + if (atomic_read(&eb->blocking_readers) || eb->blocking_writers) { write_unlock(&eb->lock); goto again; } @@ -340,7 +334,7 @@ again: */ void btrfs_tree_unlock(struct extent_buffer *eb) { - int blockers = atomic_read(&eb->blocking_writers); + int blockers = eb->blocking_writers; BUG_ON(blockers > 1); @@ -351,7 +345,7 @@ void btrfs_tree_unlock(struct extent_buffer *eb) if (blockers) { btrfs_assert_no_spinning_writers(eb); - atomic_dec(&eb->blocking_writers); + eb->blocking_writers--; /* Use the lighter barrier after atomic */ smp_mb__after_atomic(); cond_wake_up_nomb(&eb->write_lock_wq); diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 52889da69113..1744ba8b2754 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -13,6 +13,7 @@ #include "extent_io.h" #include "disk-io.h" #include "compression.h" +#include "delalloc-space.h" static struct kmem_cache *btrfs_ordered_extent_cache; @@ -924,14 +925,16 @@ out: * be reclaimed before their checksum is actually put into the btree */ int btrfs_find_ordered_sum(struct inode *inode, u64 offset, u64 disk_bytenr, - u32 *sum, int len) + u8 *sum, int len) { + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_ordered_sum *ordered_sum; struct btrfs_ordered_extent *ordered; struct btrfs_ordered_inode_tree *tree = &BTRFS_I(inode)->ordered_tree; unsigned long num_sectors; unsigned long i; u32 sectorsize = btrfs_inode_sectorsize(inode); + const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); int index = 0; ordered = btrfs_lookup_ordered_extent(inode, offset); @@ -947,10 +950,10 @@ int btrfs_find_ordered_sum(struct inode *inode, u64 offset, u64 disk_bytenr, num_sectors = ordered_sum->len >> inode->i_sb->s_blocksize_bits; num_sectors = min_t(int, len - index, num_sectors - i); - memcpy(sum + index, ordered_sum->sums + i, - num_sectors); + memcpy(sum + index, ordered_sum->sums + i * csum_size, + num_sectors * csum_size); - index += (int)num_sectors; + index += (int)num_sectors * csum_size; if (index == len) goto out; disk_bytenr += num_sectors * sectorsize; @@ -962,6 +965,51 @@ out: return index; } +/* + * btrfs_flush_ordered_range - Lock the passed range and ensures all pending + * ordered extents in it are run to completion. + * + * @tree: IO tree used for locking out other users of the range + * @inode: Inode whose ordered tree is to be searched + * @start: Beginning of range to flush + * @end: Last byte of range to lock + * @cached_state: If passed, will return the extent state responsible for the + * locked range. It's the caller's responsibility to free the cached state. + * + * This function always returns with the given range locked, ensuring after it's + * called no order extent can be pending. + */ +void btrfs_lock_and_flush_ordered_range(struct extent_io_tree *tree, + struct btrfs_inode *inode, u64 start, + u64 end, + struct extent_state **cached_state) +{ + struct btrfs_ordered_extent *ordered; + struct extent_state *cachedp = NULL; + + if (cached_state) + cachedp = *cached_state; + + while (1) { + lock_extent_bits(tree, start, end, &cachedp); + ordered = btrfs_lookup_ordered_range(inode, start, + end - start + 1); + if (!ordered) { + /* + * If no external cached_state has been passed then + * decrement the extra ref taken for cachedp since we + * aren't exposing it outside of this function + */ + if (!cached_state) + refcount_dec(&cachedp->refs); + break; + } + unlock_extent_cached(tree, start, end, &cachedp); + btrfs_start_ordered_extent(&inode->vfs_inode, ordered, 1); + btrfs_put_ordered_extent(ordered); + } +} + int __init ordered_data_init(void) { btrfs_ordered_extent_cache = kmem_cache_create("btrfs_ordered_extent", diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 4c5991c3de14..5204171ea962 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -23,7 +23,7 @@ struct btrfs_ordered_sum { int len; struct list_head list; /* last field is a variable length array of csums */ - u32 sums[]; + u8 sums[]; }; /* @@ -183,11 +183,15 @@ struct btrfs_ordered_extent *btrfs_lookup_ordered_range( int btrfs_ordered_update_i_size(struct inode *inode, u64 offset, struct btrfs_ordered_extent *ordered); int btrfs_find_ordered_sum(struct inode *inode, u64 offset, u64 disk_bytenr, - u32 *sum, int len); + u8 *sum, int len); u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr, const u64 range_start, const u64 range_len); u64 btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr, const u64 range_start, const u64 range_len); +void btrfs_lock_and_flush_ordered_range(struct extent_io_tree *tree, + struct btrfs_inode *inode, u64 start, + u64 end, + struct extent_state **cached_state); int __init ordered_data_init(void); void __cold ordered_data_exit(void); diff --git a/fs/btrfs/print-tree.c b/fs/btrfs/print-tree.c index 1141ca5fae6a..9cb50577d982 100644 --- a/fs/btrfs/print-tree.c +++ b/fs/btrfs/print-tree.c @@ -153,11 +153,11 @@ static void print_eb_refs_lock(struct extent_buffer *eb) #ifdef CONFIG_BTRFS_DEBUG btrfs_info(eb->fs_info, "refs %u lock (w:%d r:%d bw:%d br:%d sw:%d sr:%d) lock_owner %u current %u", - atomic_read(&eb->refs), atomic_read(&eb->write_locks), + atomic_read(&eb->refs), eb->write_locks, atomic_read(&eb->read_locks), - atomic_read(&eb->blocking_writers), + eb->blocking_writers, atomic_read(&eb->blocking_readers), - atomic_read(&eb->spinning_writers), + eb->spinning_writers, atomic_read(&eb->spinning_readers), eb->lock_owner, current->pid); #endif diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c index a9e2e66152ee..e0469816c678 100644 --- a/fs/btrfs/props.c +++ b/fs/btrfs/props.c @@ -257,11 +257,7 @@ static int prop_compression_validate(const char *value, size_t len) if (!value) return 0; - if (!strncmp("lzo", value, 3)) - return 0; - else if (!strncmp("zlib", value, 4)) - return 0; - else if (!strncmp("zstd", value, 4)) + if (btrfs_compress_is_valid_type(value, len)) return 0; return -EINVAL; @@ -341,7 +337,7 @@ static int inherit_props(struct btrfs_trans_handle *trans, for (i = 0; i < ARRAY_SIZE(prop_handlers); i++) { const struct prop_handler *h = &prop_handlers[i]; const char *value; - u64 num_bytes; + u64 num_bytes = 0; if (!h->inheritable) continue; diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 3e6ffbbd8b0a..f8a3c1b0a15a 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -2614,6 +2614,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, int ret = 0; int i; u64 *i_qgroups; + bool committing = false; struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_root *quota_root; struct btrfs_qgroup *srcgroup; @@ -2621,7 +2622,25 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, u32 level_size = 0; u64 nums; - mutex_lock(&fs_info->qgroup_ioctl_lock); + /* + * There are only two callers of this function. + * + * One in create_subvol() in the ioctl context, which needs to hold + * the qgroup_ioctl_lock. + * + * The other one in create_pending_snapshot() where no other qgroup + * code can modify the fs as they all need to either start a new trans + * or hold a trans handler, thus we don't need to hold + * qgroup_ioctl_lock. + * This would avoid long and complex lock chain and make lockdep happy. + */ + spin_lock(&fs_info->trans_lock); + if (trans->transaction->state == TRANS_STATE_COMMIT_DOING) + committing = true; + spin_unlock(&fs_info->trans_lock); + + if (!committing) + mutex_lock(&fs_info->qgroup_ioctl_lock); if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) goto out; @@ -2785,7 +2804,8 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, unlock: spin_unlock(&fs_info->qgroup_lock); out: - mutex_unlock(&fs_info->qgroup_ioctl_lock); + if (!committing) + mutex_unlock(&fs_info->qgroup_ioctl_lock); return ret; } diff --git a/fs/btrfs/raid56.h b/fs/btrfs/raid56.h index f5d4c13a8dbc..2503485db859 100644 --- a/fs/btrfs/raid56.h +++ b/fs/btrfs/raid56.h @@ -7,7 +7,7 @@ #ifndef BTRFS_RAID56_H #define BTRFS_RAID56_H -static inline int nr_parity_stripes(struct map_lookup *map) +static inline int nr_parity_stripes(const struct map_lookup *map) { if (map->type & BTRFS_BLOCK_GROUP_RAID5) return 1; @@ -17,7 +17,7 @@ static inline int nr_parity_stripes(struct map_lookup *map) return 0; } -static inline int nr_data_stripes(struct map_lookup *map) +static inline int nr_data_stripes(const struct map_lookup *map) { return map->num_stripes - nr_parity_stripes(map); } diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 22a3c69864fa..7f219851fa23 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -20,6 +20,7 @@ #include "inode-map.h" #include "qgroup.h" #include "print-tree.h" +#include "delalloc-space.h" /* * backref_node, mapping_node and tree_block start with this diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c index 22124122728c..47733fb55df7 100644 --- a/fs/btrfs/root-tree.c +++ b/fs/btrfs/root-tree.c @@ -9,6 +9,8 @@ #include "transaction.h" #include "disk-io.h" #include "print-tree.h" +#include "qgroup.h" +#include "space-info.h" /* * Read a root item from the tree. In case we detect a root item smaller then @@ -497,3 +499,57 @@ void btrfs_update_root_times(struct btrfs_trans_handle *trans, btrfs_set_stack_timespec_nsec(&item->ctime, ct.tv_nsec); spin_unlock(&root->root_item_lock); } + +/* + * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation + * root: the root of the parent directory + * rsv: block reservation + * items: the number of items that we need do reservation + * use_global_rsv: allow fallback to the global block reservation + * + * This function is used to reserve the space for snapshot/subvolume + * creation and deletion. Those operations are different with the + * common file/directory operations, they change two fs/file trees + * and root tree, the number of items that the qgroup reserves is + * different with the free space reservation. So we can not use + * the space reservation mechanism in start_transaction(). + */ +int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, + struct btrfs_block_rsv *rsv, int items, + bool use_global_rsv) +{ + u64 qgroup_num_bytes = 0; + u64 num_bytes; + int ret; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + + if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) { + /* One for parent inode, two for dir entries */ + qgroup_num_bytes = 3 * fs_info->nodesize; + ret = btrfs_qgroup_reserve_meta_prealloc(root, + qgroup_num_bytes, true); + if (ret) + return ret; + } + + num_bytes = btrfs_calc_trans_metadata_size(fs_info, items); + rsv->space_info = btrfs_find_space_info(fs_info, + BTRFS_BLOCK_GROUP_METADATA); + ret = btrfs_block_rsv_add(root, rsv, num_bytes, + BTRFS_RESERVE_FLUSH_ALL); + + if (ret == -ENOSPC && use_global_rsv) + ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes, true); + + if (ret && qgroup_num_bytes) + btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes); + + return ret; +} + +void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv) +{ + btrfs_block_rsv_release(fs_info, rsv, (u64)-1); +} diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index f7b29f9db5e2..0c99cf9fb595 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -6,6 +6,7 @@ #include <linux/blkdev.h> #include <linux/ratelimit.h> #include <linux/sched/mm.h> +#include <crypto/hash.h> #include "ctree.h" #include "volumes.h" #include "disk-io.h" @@ -1787,11 +1788,12 @@ static int scrub_checksum(struct scrub_block *sblock) static int scrub_checksum_data(struct scrub_block *sblock) { struct scrub_ctx *sctx = sblock->sctx; + struct btrfs_fs_info *fs_info = sctx->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); u8 csum[BTRFS_CSUM_SIZE]; u8 *on_disk_csum; struct page *page; void *buffer; - u32 crc = ~(u32)0; u64 len; int index; @@ -1799,6 +1801,9 @@ static int scrub_checksum_data(struct scrub_block *sblock) if (!sblock->pagev[0]->have_csum) return 0; + shash->tfm = fs_info->csum_shash; + crypto_shash_init(shash); + on_disk_csum = sblock->pagev[0]->csum; page = sblock->pagev[0]->page; buffer = kmap_atomic(page); @@ -1808,7 +1813,7 @@ static int scrub_checksum_data(struct scrub_block *sblock) for (;;) { u64 l = min_t(u64, len, PAGE_SIZE); - crc = btrfs_csum_data(buffer, crc, l); + crypto_shash_update(shash, buffer, l); kunmap_atomic(buffer); len -= l; if (len == 0) @@ -1820,7 +1825,7 @@ static int scrub_checksum_data(struct scrub_block *sblock) buffer = kmap_atomic(page); } - btrfs_csum_final(crc, csum); + crypto_shash_final(shash, csum); if (memcmp(csum, on_disk_csum, sctx->csum_size)) sblock->checksum_error = 1; @@ -1832,16 +1837,19 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) struct scrub_ctx *sctx = sblock->sctx; struct btrfs_header *h; struct btrfs_fs_info *fs_info = sctx->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); u8 calculated_csum[BTRFS_CSUM_SIZE]; u8 on_disk_csum[BTRFS_CSUM_SIZE]; struct page *page; void *mapped_buffer; u64 mapped_size; void *p; - u32 crc = ~(u32)0; u64 len; int index; + shash->tfm = fs_info->csum_shash; + crypto_shash_init(shash); + BUG_ON(sblock->page_count < 1); page = sblock->pagev[0]->page; mapped_buffer = kmap_atomic(page); @@ -1875,7 +1883,7 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) for (;;) { u64 l = min_t(u64, len, mapped_size); - crc = btrfs_csum_data(p, crc, l); + crypto_shash_update(shash, p, l); kunmap_atomic(mapped_buffer); len -= l; if (len == 0) @@ -1889,7 +1897,7 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) p = mapped_buffer; } - btrfs_csum_final(crc, calculated_csum); + crypto_shash_final(shash, calculated_csum); if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size)) sblock->checksum_error = 1; @@ -1900,18 +1908,22 @@ static int scrub_checksum_super(struct scrub_block *sblock) { struct btrfs_super_block *s; struct scrub_ctx *sctx = sblock->sctx; + struct btrfs_fs_info *fs_info = sctx->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); u8 calculated_csum[BTRFS_CSUM_SIZE]; u8 on_disk_csum[BTRFS_CSUM_SIZE]; struct page *page; void *mapped_buffer; u64 mapped_size; void *p; - u32 crc = ~(u32)0; int fail_gen = 0; int fail_cor = 0; u64 len; int index; + shash->tfm = fs_info->csum_shash; + crypto_shash_init(shash); + BUG_ON(sblock->page_count < 1); page = sblock->pagev[0]->page; mapped_buffer = kmap_atomic(page); @@ -1934,7 +1946,7 @@ static int scrub_checksum_super(struct scrub_block *sblock) for (;;) { u64 l = min_t(u64, len, mapped_size); - crc = btrfs_csum_data(p, crc, l); + crypto_shash_update(shash, p, l); kunmap_atomic(mapped_buffer); len -= l; if (len == 0) @@ -1948,7 +1960,7 @@ static int scrub_checksum_super(struct scrub_block *sblock) p = mapped_buffer; } - btrfs_csum_final(crc, calculated_csum); + crypto_shash_final(shash, calculated_csum); if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size)) ++fail_cor; @@ -2448,7 +2460,7 @@ static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum) ASSERT(index < UINT_MAX); num_sectors = sum->len / sctx->fs_info->sectorsize; - memcpy(csum, sum->sums + index, sctx->csum_size); + memcpy(csum, sum->sums + index * sctx->csum_size, sctx->csum_size); if (index == num_sectors - 1) { list_del(&sum->list); kfree(sum); @@ -2660,18 +2672,18 @@ static int get_raid56_logic_offset(u64 physical, int num, u64 last_offset; u32 stripe_index; u32 rot; + const int data_stripes = nr_data_stripes(map); - last_offset = (physical - map->stripes[num].physical) * - nr_data_stripes(map); + last_offset = (physical - map->stripes[num].physical) * data_stripes; if (stripe_start) *stripe_start = last_offset; *offset = last_offset; - for (i = 0; i < nr_data_stripes(map); i++) { + for (i = 0; i < data_stripes; i++) { *offset = last_offset + i * map->stripe_len; stripe_nr = div64_u64(*offset, map->stripe_len); - stripe_nr = div_u64(stripe_nr, nr_data_stripes(map)); + stripe_nr = div_u64(stripe_nr, data_stripes); /* Work out the disk rotation on this stripe-set */ stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot); @@ -3079,7 +3091,7 @@ static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, offset = map->stripe_len * (num / map->sub_stripes); increment = map->stripe_len * factor; mirror_num = num % map->sub_stripes + 1; - } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) { + } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) { increment = map->stripe_len; mirror_num = num % map->num_stripes + 1; } else if (map->type & BTRFS_BLOCK_GROUP_DUP) { @@ -3410,15 +3422,15 @@ static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, struct btrfs_block_group_cache *cache) { struct btrfs_fs_info *fs_info = sctx->fs_info; - struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree; + struct extent_map_tree *map_tree = &fs_info->mapping_tree; struct map_lookup *map; struct extent_map *em; int i; int ret = 0; - read_lock(&map_tree->map_tree.lock); - em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1); - read_unlock(&map_tree->map_tree.lock); + read_lock(&map_tree->lock); + em = lookup_extent_mapping(map_tree, chunk_offset, 1); + read_unlock(&map_tree->lock); if (!em) { /* diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index f7fe4770f0e5..69b59bf75882 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -686,7 +686,7 @@ static int send_cmd(struct send_ctx *sctx) hdr->len = cpu_to_le32(sctx->send_size - sizeof(*hdr)); hdr->crc = 0; - crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size); + crc = btrfs_crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size); hdr->crc = cpu_to_le32(crc); ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size, @@ -6929,9 +6929,23 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) if (ret) goto out; + mutex_lock(&fs_info->balance_mutex); + if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { + mutex_unlock(&fs_info->balance_mutex); + btrfs_warn_rl(fs_info, + "cannot run send because a balance operation is in progress"); + ret = -EAGAIN; + goto out; + } + fs_info->send_in_progress++; + mutex_unlock(&fs_info->balance_mutex); + current->journal_info = BTRFS_SEND_TRANS_STUB; ret = send_subvol(sctx); current->journal_info = NULL; + mutex_lock(&fs_info->balance_mutex); + fs_info->send_in_progress--; + mutex_unlock(&fs_info->balance_mutex); if (ret < 0) goto out; diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c new file mode 100644 index 000000000000..ab7b9ec4c240 --- /dev/null +++ b/fs/btrfs/space-info.c @@ -0,0 +1,1094 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "ctree.h" +#include "space-info.h" +#include "sysfs.h" +#include "volumes.h" +#include "free-space-cache.h" +#include "ordered-data.h" +#include "transaction.h" +#include "math.h" + +u64 btrfs_space_info_used(struct btrfs_space_info *s_info, + bool may_use_included) +{ + ASSERT(s_info); + return s_info->bytes_used + s_info->bytes_reserved + + s_info->bytes_pinned + s_info->bytes_readonly + + (may_use_included ? s_info->bytes_may_use : 0); +} + +/* + * after adding space to the filesystem, we need to clear the full flags + * on all the space infos. + */ +void btrfs_clear_space_info_full(struct btrfs_fs_info *info) +{ + struct list_head *head = &info->space_info; + struct btrfs_space_info *found; + + rcu_read_lock(); + list_for_each_entry_rcu(found, head, list) + found->full = 0; + rcu_read_unlock(); +} + +static const char *alloc_name(u64 flags) +{ + switch (flags) { + case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA: + return "mixed"; + case BTRFS_BLOCK_GROUP_METADATA: + return "metadata"; + case BTRFS_BLOCK_GROUP_DATA: + return "data"; + case BTRFS_BLOCK_GROUP_SYSTEM: + return "system"; + default: + WARN_ON(1); + return "invalid-combination"; + }; +} + +static int create_space_info(struct btrfs_fs_info *info, u64 flags) +{ + + struct btrfs_space_info *space_info; + int i; + int ret; + + space_info = kzalloc(sizeof(*space_info), GFP_NOFS); + if (!space_info) + return -ENOMEM; + + ret = percpu_counter_init(&space_info->total_bytes_pinned, 0, + GFP_KERNEL); + if (ret) { + kfree(space_info); + return ret; + } + + for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) + INIT_LIST_HEAD(&space_info->block_groups[i]); + init_rwsem(&space_info->groups_sem); + spin_lock_init(&space_info->lock); + space_info->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK; + space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; + init_waitqueue_head(&space_info->wait); + INIT_LIST_HEAD(&space_info->ro_bgs); + INIT_LIST_HEAD(&space_info->tickets); + INIT_LIST_HEAD(&space_info->priority_tickets); + + ret = kobject_init_and_add(&space_info->kobj, &space_info_ktype, + info->space_info_kobj, "%s", + alloc_name(space_info->flags)); + if (ret) { + kobject_put(&space_info->kobj); + return ret; + } + + list_add_rcu(&space_info->list, &info->space_info); + if (flags & BTRFS_BLOCK_GROUP_DATA) + info->data_sinfo = space_info; + + return ret; +} + +int btrfs_init_space_info(struct btrfs_fs_info *fs_info) +{ + struct btrfs_super_block *disk_super; + u64 features; + u64 flags; + int mixed = 0; + int ret; + + disk_super = fs_info->super_copy; + if (!btrfs_super_root(disk_super)) + return -EINVAL; + + features = btrfs_super_incompat_flags(disk_super); + if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) + mixed = 1; + + flags = BTRFS_BLOCK_GROUP_SYSTEM; + ret = create_space_info(fs_info, flags); + if (ret) + goto out; + + if (mixed) { + flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA; + ret = create_space_info(fs_info, flags); + } else { + flags = BTRFS_BLOCK_GROUP_METADATA; + ret = create_space_info(fs_info, flags); + if (ret) + goto out; + + flags = BTRFS_BLOCK_GROUP_DATA; + ret = create_space_info(fs_info, flags); + } +out: + return ret; +} + +void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags, + u64 total_bytes, u64 bytes_used, + u64 bytes_readonly, + struct btrfs_space_info **space_info) +{ + struct btrfs_space_info *found; + int factor; + + factor = btrfs_bg_type_to_factor(flags); + + found = btrfs_find_space_info(info, flags); + ASSERT(found); + spin_lock(&found->lock); + found->total_bytes += total_bytes; + found->disk_total += total_bytes * factor; + found->bytes_used += bytes_used; + found->disk_used += bytes_used * factor; + found->bytes_readonly += bytes_readonly; + if (total_bytes > 0) + found->full = 0; + btrfs_space_info_add_new_bytes(info, found, + total_bytes - bytes_used - + bytes_readonly); + spin_unlock(&found->lock); + *space_info = found; +} + +struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info, + u64 flags) +{ + struct list_head *head = &info->space_info; + struct btrfs_space_info *found; + + flags &= BTRFS_BLOCK_GROUP_TYPE_MASK; + + rcu_read_lock(); + list_for_each_entry_rcu(found, head, list) { + if (found->flags & flags) { + rcu_read_unlock(); + return found; + } + } + rcu_read_unlock(); + return NULL; +} + +static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global) +{ + return (global->size << 1); +} + +static int can_overcommit(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, u64 bytes, + enum btrfs_reserve_flush_enum flush, + bool system_chunk) +{ + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + u64 profile; + u64 space_size; + u64 avail; + u64 used; + int factor; + + /* Don't overcommit when in mixed mode. */ + if (space_info->flags & BTRFS_BLOCK_GROUP_DATA) + return 0; + + if (system_chunk) + profile = btrfs_system_alloc_profile(fs_info); + else + profile = btrfs_metadata_alloc_profile(fs_info); + + used = btrfs_space_info_used(space_info, false); + + /* + * We only want to allow over committing if we have lots of actual space + * free, but if we don't have enough space to handle the global reserve + * space then we could end up having a real enospc problem when trying + * to allocate a chunk or some other such important allocation. + */ + spin_lock(&global_rsv->lock); + space_size = calc_global_rsv_need_space(global_rsv); + spin_unlock(&global_rsv->lock); + if (used + space_size >= space_info->total_bytes) + return 0; + + used += space_info->bytes_may_use; + + avail = atomic64_read(&fs_info->free_chunk_space); + + /* + * If we have dup, raid1 or raid10 then only half of the free + * space is actually usable. For raid56, the space info used + * doesn't include the parity drive, so we don't have to + * change the math + */ + factor = btrfs_bg_type_to_factor(profile); + avail = div_u64(avail, factor); + + /* + * If we aren't flushing all things, let us overcommit up to + * 1/2th of the space. If we can flush, don't let us overcommit + * too much, let it overcommit up to 1/8 of the space. + */ + if (flush == BTRFS_RESERVE_FLUSH_ALL) + avail >>= 3; + else + avail >>= 1; + + if (used + bytes < space_info->total_bytes + avail) + return 1; + return 0; +} + +/* + * This is for space we already have accounted in space_info->bytes_may_use, so + * basically when we're returning space from block_rsv's. + */ +void btrfs_space_info_add_old_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + u64 num_bytes) +{ + struct reserve_ticket *ticket; + struct list_head *head; + u64 used; + enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH; + bool check_overcommit = false; + + spin_lock(&space_info->lock); + head = &space_info->priority_tickets; + + /* + * If we are over our limit then we need to check and see if we can + * overcommit, and if we can't then we just need to free up our space + * and not satisfy any requests. + */ + used = btrfs_space_info_used(space_info, true); + if (used - num_bytes >= space_info->total_bytes) + check_overcommit = true; +again: + while (!list_empty(head) && num_bytes) { + ticket = list_first_entry(head, struct reserve_ticket, + list); + /* + * We use 0 bytes because this space is already reserved, so + * adding the ticket space would be a double count. + */ + if (check_overcommit && + !can_overcommit(fs_info, space_info, 0, flush, false)) + break; + if (num_bytes >= ticket->bytes) { + list_del_init(&ticket->list); + num_bytes -= ticket->bytes; + ticket->bytes = 0; + space_info->tickets_id++; + wake_up(&ticket->wait); + } else { + ticket->bytes -= num_bytes; + num_bytes = 0; + } + } + + if (num_bytes && head == &space_info->priority_tickets) { + head = &space_info->tickets; + flush = BTRFS_RESERVE_FLUSH_ALL; + goto again; + } + btrfs_space_info_update_bytes_may_use(fs_info, space_info, -num_bytes); + trace_btrfs_space_reservation(fs_info, "space_info", + space_info->flags, num_bytes, 0); + spin_unlock(&space_info->lock); +} + +/* + * This is for newly allocated space that isn't accounted in + * space_info->bytes_may_use yet. So if we allocate a chunk or unpin an extent + * we use this helper. + */ +void btrfs_space_info_add_new_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + u64 num_bytes) +{ + struct reserve_ticket *ticket; + struct list_head *head = &space_info->priority_tickets; + +again: + while (!list_empty(head) && num_bytes) { + ticket = list_first_entry(head, struct reserve_ticket, + list); + if (num_bytes >= ticket->bytes) { + trace_btrfs_space_reservation(fs_info, "space_info", + space_info->flags, + ticket->bytes, 1); + list_del_init(&ticket->list); + num_bytes -= ticket->bytes; + btrfs_space_info_update_bytes_may_use(fs_info, + space_info, + ticket->bytes); + ticket->bytes = 0; + space_info->tickets_id++; + wake_up(&ticket->wait); + } else { + trace_btrfs_space_reservation(fs_info, "space_info", + space_info->flags, + num_bytes, 1); + btrfs_space_info_update_bytes_may_use(fs_info, + space_info, + num_bytes); + ticket->bytes -= num_bytes; + num_bytes = 0; + } + } + + if (num_bytes && head == &space_info->priority_tickets) { + head = &space_info->tickets; + goto again; + } +} + +#define DUMP_BLOCK_RSV(fs_info, rsv_name) \ +do { \ + struct btrfs_block_rsv *__rsv = &(fs_info)->rsv_name; \ + spin_lock(&__rsv->lock); \ + btrfs_info(fs_info, #rsv_name ": size %llu reserved %llu", \ + __rsv->size, __rsv->reserved); \ + spin_unlock(&__rsv->lock); \ +} while (0) + +void btrfs_dump_space_info(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *info, u64 bytes, + int dump_block_groups) +{ + struct btrfs_block_group_cache *cache; + int index = 0; + + spin_lock(&info->lock); + btrfs_info(fs_info, "space_info %llu has %llu free, is %sfull", + info->flags, + info->total_bytes - btrfs_space_info_used(info, true), + info->full ? "" : "not "); + btrfs_info(fs_info, + "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu", + info->total_bytes, info->bytes_used, info->bytes_pinned, + info->bytes_reserved, info->bytes_may_use, + info->bytes_readonly); + spin_unlock(&info->lock); + + DUMP_BLOCK_RSV(fs_info, global_block_rsv); + DUMP_BLOCK_RSV(fs_info, trans_block_rsv); + DUMP_BLOCK_RSV(fs_info, chunk_block_rsv); + DUMP_BLOCK_RSV(fs_info, delayed_block_rsv); + DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv); + + if (!dump_block_groups) + return; + + down_read(&info->groups_sem); +again: + list_for_each_entry(cache, &info->block_groups[index], list) { + spin_lock(&cache->lock); + btrfs_info(fs_info, + "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s", + cache->key.objectid, cache->key.offset, + btrfs_block_group_used(&cache->item), cache->pinned, + cache->reserved, cache->ro ? "[readonly]" : ""); + btrfs_dump_free_space(cache, bytes); + spin_unlock(&cache->lock); + } + if (++index < BTRFS_NR_RAID_TYPES) + goto again; + up_read(&info->groups_sem); +} + +static void btrfs_writeback_inodes_sb_nr(struct btrfs_fs_info *fs_info, + unsigned long nr_pages, int nr_items) +{ + struct super_block *sb = fs_info->sb; + + if (down_read_trylock(&sb->s_umount)) { + writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE); + up_read(&sb->s_umount); + } else { + /* + * We needn't worry the filesystem going from r/w to r/o though + * we don't acquire ->s_umount mutex, because the filesystem + * should guarantee the delalloc inodes list be empty after + * the filesystem is readonly(all dirty pages are written to + * the disk). + */ + btrfs_start_delalloc_roots(fs_info, nr_items); + if (!current->journal_info) + btrfs_wait_ordered_roots(fs_info, nr_items, 0, (u64)-1); + } +} + +static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info, + u64 to_reclaim) +{ + u64 bytes; + u64 nr; + + bytes = btrfs_calc_trans_metadata_size(fs_info, 1); + nr = div64_u64(to_reclaim, bytes); + if (!nr) + nr = 1; + return nr; +} + +#define EXTENT_SIZE_PER_ITEM SZ_256K + +/* + * shrink metadata reservation for delalloc + */ +static void shrink_delalloc(struct btrfs_fs_info *fs_info, u64 to_reclaim, + u64 orig, bool wait_ordered) +{ + struct btrfs_space_info *space_info; + struct btrfs_trans_handle *trans; + u64 delalloc_bytes; + u64 dio_bytes; + u64 async_pages; + u64 items; + long time_left; + unsigned long nr_pages; + int loops; + + /* Calc the number of the pages we need flush for space reservation */ + items = calc_reclaim_items_nr(fs_info, to_reclaim); + to_reclaim = items * EXTENT_SIZE_PER_ITEM; + + trans = (struct btrfs_trans_handle *)current->journal_info; + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); + + delalloc_bytes = percpu_counter_sum_positive( + &fs_info->delalloc_bytes); + dio_bytes = percpu_counter_sum_positive(&fs_info->dio_bytes); + if (delalloc_bytes == 0 && dio_bytes == 0) { + if (trans) + return; + if (wait_ordered) + btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1); + return; + } + + /* + * If we are doing more ordered than delalloc we need to just wait on + * ordered extents, otherwise we'll waste time trying to flush delalloc + * that likely won't give us the space back we need. + */ + if (dio_bytes > delalloc_bytes) + wait_ordered = true; + + loops = 0; + while ((delalloc_bytes || dio_bytes) && loops < 3) { + nr_pages = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT; + + /* + * Triggers inode writeback for up to nr_pages. This will invoke + * ->writepages callback and trigger delalloc filling + * (btrfs_run_delalloc_range()). + */ + btrfs_writeback_inodes_sb_nr(fs_info, nr_pages, items); + + /* + * We need to wait for the compressed pages to start before + * we continue. + */ + async_pages = atomic_read(&fs_info->async_delalloc_pages); + if (!async_pages) + goto skip_async; + + /* + * Calculate how many compressed pages we want to be written + * before we continue. I.e if there are more async pages than we + * require wait_event will wait until nr_pages are written. + */ + if (async_pages <= nr_pages) + async_pages = 0; + else + async_pages -= nr_pages; + + wait_event(fs_info->async_submit_wait, + atomic_read(&fs_info->async_delalloc_pages) <= + (int)async_pages); +skip_async: + spin_lock(&space_info->lock); + if (list_empty(&space_info->tickets) && + list_empty(&space_info->priority_tickets)) { + spin_unlock(&space_info->lock); + break; + } + spin_unlock(&space_info->lock); + + loops++; + if (wait_ordered && !trans) { + btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1); + } else { + time_left = schedule_timeout_killable(1); + if (time_left) + break; + } + delalloc_bytes = percpu_counter_sum_positive( + &fs_info->delalloc_bytes); + dio_bytes = percpu_counter_sum_positive(&fs_info->dio_bytes); + } +} + +/** + * maybe_commit_transaction - possibly commit the transaction if its ok to + * @root - the root we're allocating for + * @bytes - the number of bytes we want to reserve + * @force - force the commit + * + * This will check to make sure that committing the transaction will actually + * get us somewhere and then commit the transaction if it does. Otherwise it + * will return -ENOSPC. + */ +static int may_commit_transaction(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info) +{ + struct reserve_ticket *ticket = NULL; + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv; + struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_trans_handle *trans; + u64 bytes_needed; + u64 reclaim_bytes = 0; + + trans = (struct btrfs_trans_handle *)current->journal_info; + if (trans) + return -EAGAIN; + + spin_lock(&space_info->lock); + if (!list_empty(&space_info->priority_tickets)) + ticket = list_first_entry(&space_info->priority_tickets, + struct reserve_ticket, list); + else if (!list_empty(&space_info->tickets)) + ticket = list_first_entry(&space_info->tickets, + struct reserve_ticket, list); + bytes_needed = (ticket) ? ticket->bytes : 0; + spin_unlock(&space_info->lock); + + if (!bytes_needed) + return 0; + + trans = btrfs_join_transaction(fs_info->extent_root); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + /* + * See if there is enough pinned space to make this reservation, or if + * we have block groups that are going to be freed, allowing us to + * possibly do a chunk allocation the next loop through. + */ + if (test_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags) || + __percpu_counter_compare(&space_info->total_bytes_pinned, + bytes_needed, + BTRFS_TOTAL_BYTES_PINNED_BATCH) >= 0) + goto commit; + + /* + * See if there is some space in the delayed insertion reservation for + * this reservation. + */ + if (space_info != delayed_rsv->space_info) + goto enospc; + + spin_lock(&delayed_rsv->lock); + reclaim_bytes += delayed_rsv->reserved; + spin_unlock(&delayed_rsv->lock); + + spin_lock(&delayed_refs_rsv->lock); + reclaim_bytes += delayed_refs_rsv->reserved; + spin_unlock(&delayed_refs_rsv->lock); + if (reclaim_bytes >= bytes_needed) + goto commit; + bytes_needed -= reclaim_bytes; + + if (__percpu_counter_compare(&space_info->total_bytes_pinned, + bytes_needed, + BTRFS_TOTAL_BYTES_PINNED_BATCH) < 0) + goto enospc; + +commit: + return btrfs_commit_transaction(trans); +enospc: + btrfs_end_transaction(trans); + return -ENOSPC; +} + +/* + * Try to flush some data based on policy set by @state. This is only advisory + * and may fail for various reasons. The caller is supposed to examine the + * state of @space_info to detect the outcome. + */ +static void flush_space(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, u64 num_bytes, + int state) +{ + struct btrfs_root *root = fs_info->extent_root; + struct btrfs_trans_handle *trans; + int nr; + int ret = 0; + + switch (state) { + case FLUSH_DELAYED_ITEMS_NR: + case FLUSH_DELAYED_ITEMS: + if (state == FLUSH_DELAYED_ITEMS_NR) + nr = calc_reclaim_items_nr(fs_info, num_bytes) * 2; + else + nr = -1; + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + ret = btrfs_run_delayed_items_nr(trans, nr); + btrfs_end_transaction(trans); + break; + case FLUSH_DELALLOC: + case FLUSH_DELALLOC_WAIT: + shrink_delalloc(fs_info, num_bytes * 2, num_bytes, + state == FLUSH_DELALLOC_WAIT); + break; + case FLUSH_DELAYED_REFS_NR: + case FLUSH_DELAYED_REFS: + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + if (state == FLUSH_DELAYED_REFS_NR) + nr = calc_reclaim_items_nr(fs_info, num_bytes); + else + nr = 0; + btrfs_run_delayed_refs(trans, nr); + btrfs_end_transaction(trans); + break; + case ALLOC_CHUNK: + case ALLOC_CHUNK_FORCE: + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + ret = btrfs_chunk_alloc(trans, + btrfs_metadata_alloc_profile(fs_info), + (state == ALLOC_CHUNK) ? CHUNK_ALLOC_NO_FORCE : + CHUNK_ALLOC_FORCE); + btrfs_end_transaction(trans); + if (ret > 0 || ret == -ENOSPC) + ret = 0; + break; + case COMMIT_TRANS: + /* + * If we have pending delayed iputs then we could free up a + * bunch of pinned space, so make sure we run the iputs before + * we do our pinned bytes check below. + */ + btrfs_run_delayed_iputs(fs_info); + btrfs_wait_on_delayed_iputs(fs_info); + + ret = may_commit_transaction(fs_info, space_info); + break; + default: + ret = -ENOSPC; + break; + } + + trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, state, + ret); + return; +} + +static inline u64 +btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + bool system_chunk) +{ + struct reserve_ticket *ticket; + u64 used; + u64 expected; + u64 to_reclaim = 0; + + list_for_each_entry(ticket, &space_info->tickets, list) + to_reclaim += ticket->bytes; + list_for_each_entry(ticket, &space_info->priority_tickets, list) + to_reclaim += ticket->bytes; + if (to_reclaim) + return to_reclaim; + + to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M); + if (can_overcommit(fs_info, space_info, to_reclaim, + BTRFS_RESERVE_FLUSH_ALL, system_chunk)) + return 0; + + used = btrfs_space_info_used(space_info, true); + + if (can_overcommit(fs_info, space_info, SZ_1M, + BTRFS_RESERVE_FLUSH_ALL, system_chunk)) + expected = div_factor_fine(space_info->total_bytes, 95); + else + expected = div_factor_fine(space_info->total_bytes, 90); + + if (used > expected) + to_reclaim = used - expected; + else + to_reclaim = 0; + to_reclaim = min(to_reclaim, space_info->bytes_may_use + + space_info->bytes_reserved); + return to_reclaim; +} + +static inline int need_do_async_reclaim(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + u64 used, bool system_chunk) +{ + u64 thresh = div_factor_fine(space_info->total_bytes, 98); + + /* If we're just plain full then async reclaim just slows us down. */ + if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) + return 0; + + if (!btrfs_calc_reclaim_metadata_size(fs_info, space_info, + system_chunk)) + return 0; + + return (used >= thresh && !btrfs_fs_closing(fs_info) && + !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); +} + +static bool wake_all_tickets(struct list_head *head) +{ + struct reserve_ticket *ticket; + + while (!list_empty(head)) { + ticket = list_first_entry(head, struct reserve_ticket, list); + list_del_init(&ticket->list); + ticket->error = -ENOSPC; + wake_up(&ticket->wait); + if (ticket->bytes != ticket->orig_bytes) + return true; + } + return false; +} + +/* + * This is for normal flushers, we can wait all goddamned day if we want to. We + * will loop and continuously try to flush as long as we are making progress. + * We count progress as clearing off tickets each time we have to loop. + */ +static void btrfs_async_reclaim_metadata_space(struct work_struct *work) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_space_info *space_info; + u64 to_reclaim; + int flush_state; + int commit_cycles = 0; + u64 last_tickets_id; + + fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work); + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); + + spin_lock(&space_info->lock); + to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info, + false); + if (!to_reclaim) { + space_info->flush = 0; + spin_unlock(&space_info->lock); + return; + } + last_tickets_id = space_info->tickets_id; + spin_unlock(&space_info->lock); + + flush_state = FLUSH_DELAYED_ITEMS_NR; + do { + flush_space(fs_info, space_info, to_reclaim, flush_state); + spin_lock(&space_info->lock); + if (list_empty(&space_info->tickets)) { + space_info->flush = 0; + spin_unlock(&space_info->lock); + return; + } + to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, + space_info, + false); + if (last_tickets_id == space_info->tickets_id) { + flush_state++; + } else { + last_tickets_id = space_info->tickets_id; + flush_state = FLUSH_DELAYED_ITEMS_NR; + if (commit_cycles) + commit_cycles--; + } + + /* + * We don't want to force a chunk allocation until we've tried + * pretty hard to reclaim space. Think of the case where we + * freed up a bunch of space and so have a lot of pinned space + * to reclaim. We would rather use that than possibly create a + * underutilized metadata chunk. So if this is our first run + * through the flushing state machine skip ALLOC_CHUNK_FORCE and + * commit the transaction. If nothing has changed the next go + * around then we can force a chunk allocation. + */ + if (flush_state == ALLOC_CHUNK_FORCE && !commit_cycles) + flush_state++; + + if (flush_state > COMMIT_TRANS) { + commit_cycles++; + if (commit_cycles > 2) { + if (wake_all_tickets(&space_info->tickets)) { + flush_state = FLUSH_DELAYED_ITEMS_NR; + commit_cycles--; + } else { + space_info->flush = 0; + } + } else { + flush_state = FLUSH_DELAYED_ITEMS_NR; + } + } + spin_unlock(&space_info->lock); + } while (flush_state <= COMMIT_TRANS); +} + +void btrfs_init_async_reclaim_work(struct work_struct *work) +{ + INIT_WORK(work, btrfs_async_reclaim_metadata_space); +} + +static const enum btrfs_flush_state priority_flush_states[] = { + FLUSH_DELAYED_ITEMS_NR, + FLUSH_DELAYED_ITEMS, + ALLOC_CHUNK, +}; + +static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + struct reserve_ticket *ticket) +{ + u64 to_reclaim; + int flush_state; + + spin_lock(&space_info->lock); + to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info, + false); + if (!to_reclaim) { + spin_unlock(&space_info->lock); + return; + } + spin_unlock(&space_info->lock); + + flush_state = 0; + do { + flush_space(fs_info, space_info, to_reclaim, + priority_flush_states[flush_state]); + flush_state++; + spin_lock(&space_info->lock); + if (ticket->bytes == 0) { + spin_unlock(&space_info->lock); + return; + } + spin_unlock(&space_info->lock); + } while (flush_state < ARRAY_SIZE(priority_flush_states)); +} + +static int wait_reserve_ticket(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + struct reserve_ticket *ticket) + +{ + DEFINE_WAIT(wait); + u64 reclaim_bytes = 0; + int ret = 0; + + spin_lock(&space_info->lock); + while (ticket->bytes > 0 && ticket->error == 0) { + ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE); + if (ret) { + ret = -EINTR; + break; + } + spin_unlock(&space_info->lock); + + schedule(); + + finish_wait(&ticket->wait, &wait); + spin_lock(&space_info->lock); + } + if (!ret) + ret = ticket->error; + if (!list_empty(&ticket->list)) + list_del_init(&ticket->list); + if (ticket->bytes && ticket->bytes < ticket->orig_bytes) + reclaim_bytes = ticket->orig_bytes - ticket->bytes; + spin_unlock(&space_info->lock); + + if (reclaim_bytes) + btrfs_space_info_add_old_bytes(fs_info, space_info, + reclaim_bytes); + return ret; +} + +/** + * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space + * @root - the root we're allocating for + * @space_info - the space info we want to allocate from + * @orig_bytes - the number of bytes we want + * @flush - whether or not we can flush to make our reservation + * + * This will reserve orig_bytes number of bytes from the space info associated + * with the block_rsv. If there is not enough space it will make an attempt to + * flush out space to make room. It will do this by flushing delalloc if + * possible or committing the transaction. If flush is 0 then no attempts to + * regain reservations will be made and this will fail if there is not enough + * space already. + */ +static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + u64 orig_bytes, + enum btrfs_reserve_flush_enum flush, + bool system_chunk) +{ + struct reserve_ticket ticket; + u64 used; + u64 reclaim_bytes = 0; + int ret = 0; + + ASSERT(orig_bytes); + ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_ALL); + + spin_lock(&space_info->lock); + ret = -ENOSPC; + used = btrfs_space_info_used(space_info, true); + + /* + * Carry on if we have enough space (short-circuit) OR call + * can_overcommit() to ensure we can overcommit to continue. + */ + if ((used + orig_bytes <= space_info->total_bytes) || + can_overcommit(fs_info, space_info, orig_bytes, flush, + system_chunk)) { + btrfs_space_info_update_bytes_may_use(fs_info, space_info, + orig_bytes); + trace_btrfs_space_reservation(fs_info, "space_info", + space_info->flags, orig_bytes, 1); + ret = 0; + } + + /* + * If we couldn't make a reservation then setup our reservation ticket + * and kick the async worker if it's not already running. + * + * If we are a priority flusher then we just need to add our ticket to + * the list and we will do our own flushing further down. + */ + if (ret && flush != BTRFS_RESERVE_NO_FLUSH) { + ticket.orig_bytes = orig_bytes; + ticket.bytes = orig_bytes; + ticket.error = 0; + init_waitqueue_head(&ticket.wait); + if (flush == BTRFS_RESERVE_FLUSH_ALL) { + list_add_tail(&ticket.list, &space_info->tickets); + if (!space_info->flush) { + space_info->flush = 1; + trace_btrfs_trigger_flush(fs_info, + space_info->flags, + orig_bytes, flush, + "enospc"); + queue_work(system_unbound_wq, + &fs_info->async_reclaim_work); + } + } else { + list_add_tail(&ticket.list, + &space_info->priority_tickets); + } + } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { + used += orig_bytes; + /* + * We will do the space reservation dance during log replay, + * which means we won't have fs_info->fs_root set, so don't do + * the async reclaim as we will panic. + */ + if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && + need_do_async_reclaim(fs_info, space_info, + used, system_chunk) && + !work_busy(&fs_info->async_reclaim_work)) { + trace_btrfs_trigger_flush(fs_info, space_info->flags, + orig_bytes, flush, "preempt"); + queue_work(system_unbound_wq, + &fs_info->async_reclaim_work); + } + } + spin_unlock(&space_info->lock); + if (!ret || flush == BTRFS_RESERVE_NO_FLUSH) + return ret; + + if (flush == BTRFS_RESERVE_FLUSH_ALL) + return wait_reserve_ticket(fs_info, space_info, &ticket); + + ret = 0; + priority_reclaim_metadata_space(fs_info, space_info, &ticket); + spin_lock(&space_info->lock); + if (ticket.bytes) { + if (ticket.bytes < orig_bytes) + reclaim_bytes = orig_bytes - ticket.bytes; + list_del_init(&ticket.list); + ret = -ENOSPC; + } + spin_unlock(&space_info->lock); + + if (reclaim_bytes) + btrfs_space_info_add_old_bytes(fs_info, space_info, + reclaim_bytes); + ASSERT(list_empty(&ticket.list)); + return ret; +} + +/** + * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space + * @root - the root we're allocating for + * @block_rsv - the block_rsv we're allocating for + * @orig_bytes - the number of bytes we want + * @flush - whether or not we can flush to make our reservation + * + * This will reserve orig_bytes number of bytes from the space info associated + * with the block_rsv. If there is not enough space it will make an attempt to + * flush out space to make room. It will do this by flushing delalloc if + * possible or committing the transaction. If flush is 0 then no attempts to + * regain reservations will be made and this will fail if there is not enough + * space already. + */ +int btrfs_reserve_metadata_bytes(struct btrfs_root *root, + struct btrfs_block_rsv *block_rsv, + u64 orig_bytes, + enum btrfs_reserve_flush_enum flush) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + int ret; + bool system_chunk = (root == fs_info->chunk_root); + + ret = __reserve_metadata_bytes(fs_info, block_rsv->space_info, + orig_bytes, flush, system_chunk); + if (ret == -ENOSPC && + unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) { + if (block_rsv != global_rsv && + !btrfs_block_rsv_use_bytes(global_rsv, orig_bytes)) + ret = 0; + } + if (ret == -ENOSPC) { + trace_btrfs_space_reservation(fs_info, "space_info:enospc", + block_rsv->space_info->flags, + orig_bytes, 1); + + if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) + btrfs_dump_space_info(fs_info, block_rsv->space_info, + orig_bytes, 0); + } + return ret; +} diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h new file mode 100644 index 000000000000..c2b54b8e1a14 --- /dev/null +++ b/fs/btrfs/space-info.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_SPACE_INFO_H +#define BTRFS_SPACE_INFO_H + +struct btrfs_space_info { + spinlock_t lock; + + u64 total_bytes; /* total bytes in the space, + this doesn't take mirrors into account */ + u64 bytes_used; /* total bytes used, + this doesn't take mirrors into account */ + u64 bytes_pinned; /* total bytes pinned, will be freed when the + transaction finishes */ + u64 bytes_reserved; /* total bytes the allocator has reserved for + current allocations */ + u64 bytes_may_use; /* number of bytes that may be used for + delalloc/allocations */ + u64 bytes_readonly; /* total bytes that are read only */ + + u64 max_extent_size; /* This will hold the maximum extent size of + the space info if we had an ENOSPC in the + allocator. */ + + unsigned int full:1; /* indicates that we cannot allocate any more + chunks for this space */ + unsigned int chunk_alloc:1; /* set if we are allocating a chunk */ + + unsigned int flush:1; /* set if we are trying to make space */ + + unsigned int force_alloc; /* set if we need to force a chunk + alloc for this space */ + + u64 disk_used; /* total bytes used on disk */ + u64 disk_total; /* total bytes on disk, takes mirrors into + account */ + + u64 flags; + + /* + * bytes_pinned is kept in line with what is actually pinned, as in + * we've called update_block_group and dropped the bytes_used counter + * and increased the bytes_pinned counter. However this means that + * bytes_pinned does not reflect the bytes that will be pinned once the + * delayed refs are flushed, so this counter is inc'ed every time we + * call btrfs_free_extent so it is a realtime count of what will be + * freed once the transaction is committed. It will be zeroed every + * time the transaction commits. + */ + struct percpu_counter total_bytes_pinned; + + struct list_head list; + /* Protected by the spinlock 'lock'. */ + struct list_head ro_bgs; + struct list_head priority_tickets; + struct list_head tickets; + /* + * tickets_id just indicates the next ticket will be handled, so note + * it's not stored per ticket. + */ + u64 tickets_id; + + struct rw_semaphore groups_sem; + /* for block groups in our same type */ + struct list_head block_groups[BTRFS_NR_RAID_TYPES]; + wait_queue_head_t wait; + + struct kobject kobj; + struct kobject *block_group_kobjs[BTRFS_NR_RAID_TYPES]; +}; + +struct reserve_ticket { + u64 orig_bytes; + u64 bytes; + int error; + struct list_head list; + wait_queue_head_t wait; +}; + +static inline bool btrfs_mixed_space_info(struct btrfs_space_info *space_info) +{ + return ((space_info->flags & BTRFS_BLOCK_GROUP_METADATA) && + (space_info->flags & BTRFS_BLOCK_GROUP_DATA)); +} + +/* + * + * Declare a helper function to detect underflow of various space info members + */ +#define DECLARE_SPACE_INFO_UPDATE(name) \ +static inline void \ +btrfs_space_info_update_##name(struct btrfs_fs_info *fs_info, \ + struct btrfs_space_info *sinfo, \ + s64 bytes) \ +{ \ + lockdep_assert_held(&sinfo->lock); \ + trace_update_##name(fs_info, sinfo, sinfo->name, bytes); \ + if (bytes < 0 && sinfo->name < -bytes) { \ + WARN_ON(1); \ + sinfo->name = 0; \ + return; \ + } \ + sinfo->name += bytes; \ +} + +DECLARE_SPACE_INFO_UPDATE(bytes_may_use); +DECLARE_SPACE_INFO_UPDATE(bytes_pinned); + +void btrfs_space_info_add_new_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + u64 num_bytes); +void btrfs_space_info_add_old_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + u64 num_bytes); +int btrfs_init_space_info(struct btrfs_fs_info *fs_info); +void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags, + u64 total_bytes, u64 bytes_used, + u64 bytes_readonly, + struct btrfs_space_info **space_info); +struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info, + u64 flags); +u64 btrfs_space_info_used(struct btrfs_space_info *s_info, + bool may_use_included); +void btrfs_clear_space_info_full(struct btrfs_fs_info *info); +void btrfs_dump_space_info(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *info, u64 bytes, + int dump_block_groups); +int btrfs_reserve_metadata_bytes(struct btrfs_root *root, + struct btrfs_block_rsv *block_rsv, + u64 orig_bytes, + enum btrfs_reserve_flush_enum flush); + +#endif /* BTRFS_SPACE_INFO_H */ diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 0645ec428b4f..78de9d5d80c6 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -42,6 +42,7 @@ #include "dev-replace.h" #include "free-space-cache.h" #include "backref.h" +#include "space-info.h" #include "tests/btrfs-tests.h" #include "qgroup.h" @@ -1553,6 +1554,8 @@ static struct dentry *btrfs_mount_root(struct file_system_type *fs_type, } else { snprintf(s->s_id, sizeof(s->s_id), "%pg", bdev); btrfs_sb(s)->bdev_holder = fs_type; + if (!strstr(crc32c_impl(), "generic")) + set_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags); error = btrfs_fill_super(s, fs_devices, data); } if (!error) @@ -1601,14 +1604,10 @@ static struct dentry *btrfs_mount(struct file_system_type *fs_type, int flags, { struct vfsmount *mnt_root; struct dentry *root; - fmode_t mode = FMODE_READ; char *subvol_name = NULL; u64 subvol_objectid = 0; int error = 0; - if (!(flags & SB_RDONLY)) - mode |= FMODE_WRITE; - error = btrfs_parse_subvol_options(data, &subvol_name, &subvol_objectid); if (error) { @@ -1904,8 +1903,9 @@ static inline int btrfs_calc_avail_data_space(struct btrfs_fs_info *fs_info, u64 type; u64 avail_space; u64 min_stripe_size; - int min_stripes = 1, num_stripes = 1; + int min_stripes, num_stripes = 1; int i = 0, nr_devices; + const struct btrfs_raid_attr *rattr; /* * We aren't under the device list lock, so this is racy-ish, but good @@ -1929,21 +1929,18 @@ static inline int btrfs_calc_avail_data_space(struct btrfs_fs_info *fs_info, /* calc min stripe number for data space allocation */ type = btrfs_data_alloc_profile(fs_info); - if (type & BTRFS_BLOCK_GROUP_RAID0) { - min_stripes = 2; + rattr = &btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)]; + min_stripes = rattr->devs_min; + + if (type & BTRFS_BLOCK_GROUP_RAID0) num_stripes = nr_devices; - } else if (type & BTRFS_BLOCK_GROUP_RAID1) { - min_stripes = 2; + else if (type & BTRFS_BLOCK_GROUP_RAID1) num_stripes = 2; - } else if (type & BTRFS_BLOCK_GROUP_RAID10) { - min_stripes = 4; + else if (type & BTRFS_BLOCK_GROUP_RAID10) num_stripes = 4; - } - if (type & BTRFS_BLOCK_GROUP_DUP) - min_stripe_size = 2 * BTRFS_STRIPE_LEN; - else - min_stripe_size = BTRFS_STRIPE_LEN; + /* Adjust for more than 1 stripe per device */ + min_stripe_size = rattr->dev_stripes * BTRFS_STRIPE_LEN; rcu_read_lock(); list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) { @@ -2466,3 +2463,4 @@ late_initcall(init_btrfs_fs); module_exit(exit_btrfs_fs) MODULE_LICENSE("GPL"); +MODULE_SOFTDEP("pre: crc32c"); diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index c1dfc97893ba..9539f8143b7a 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -16,6 +16,7 @@ #include "transaction.h" #include "sysfs.h" #include "volumes.h" +#include "space-info.h" static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj); static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj); diff --git a/fs/btrfs/tests/extent-io-tests.c b/fs/btrfs/tests/extent-io-tests.c index 7bf4d5734dbe..1bf6b5a79191 100644 --- a/fs/btrfs/tests/extent-io-tests.c +++ b/fs/btrfs/tests/extent-io-tests.c @@ -10,6 +10,7 @@ #include "btrfs-tests.h" #include "../ctree.h" #include "../extent_io.h" +#include "../btrfs_inode.h" #define PROCESS_UNLOCK (1 << 0) #define PROCESS_RELEASE (1 << 1) @@ -58,7 +59,7 @@ static noinline int process_page_range(struct inode *inode, u64 start, u64 end, static int test_find_delalloc(u32 sectorsize) { struct inode *inode; - struct extent_io_tree tmp; + struct extent_io_tree *tmp; struct page *page; struct page *locked_page = NULL; unsigned long index = 0; @@ -76,12 +77,13 @@ static int test_find_delalloc(u32 sectorsize) test_std_err(TEST_ALLOC_INODE); return -ENOMEM; } + tmp = &BTRFS_I(inode)->io_tree; /* * Passing NULL as we don't have fs_info but tracepoints are not used * at this point */ - extent_io_tree_init(NULL, &tmp, IO_TREE_SELFTEST, NULL); + extent_io_tree_init(NULL, tmp, IO_TREE_SELFTEST, NULL); /* * First go through and create and mark all of our pages dirty, we pin @@ -108,10 +110,10 @@ static int test_find_delalloc(u32 sectorsize) * |--- delalloc ---| * |--- search ---| */ - set_extent_delalloc(&tmp, 0, sectorsize - 1, 0, NULL); + set_extent_delalloc(tmp, 0, sectorsize - 1, 0, NULL); start = 0; end = 0; - found = find_lock_delalloc_range(inode, &tmp, locked_page, &start, + found = find_lock_delalloc_range(inode, locked_page, &start, &end); if (!found) { test_err("should have found at least one delalloc"); @@ -122,7 +124,7 @@ static int test_find_delalloc(u32 sectorsize) sectorsize - 1, start, end); goto out_bits; } - unlock_extent(&tmp, start, end); + unlock_extent(tmp, start, end); unlock_page(locked_page); put_page(locked_page); @@ -139,10 +141,10 @@ static int test_find_delalloc(u32 sectorsize) test_err("couldn't find the locked page"); goto out_bits; } - set_extent_delalloc(&tmp, sectorsize, max_bytes - 1, 0, NULL); + set_extent_delalloc(tmp, sectorsize, max_bytes - 1, 0, NULL); start = test_start; end = 0; - found = find_lock_delalloc_range(inode, &tmp, locked_page, &start, + found = find_lock_delalloc_range(inode, locked_page, &start, &end); if (!found) { test_err("couldn't find delalloc in our range"); @@ -158,7 +160,7 @@ static int test_find_delalloc(u32 sectorsize) test_err("there were unlocked pages in the range"); goto out_bits; } - unlock_extent(&tmp, start, end); + unlock_extent(tmp, start, end); /* locked_page was unlocked above */ put_page(locked_page); @@ -176,7 +178,7 @@ static int test_find_delalloc(u32 sectorsize) } start = test_start; end = 0; - found = find_lock_delalloc_range(inode, &tmp, locked_page, &start, + found = find_lock_delalloc_range(inode, locked_page, &start, &end); if (found) { test_err("found range when we shouldn't have"); @@ -194,10 +196,10 @@ static int test_find_delalloc(u32 sectorsize) * * We are re-using our test_start from above since it works out well. */ - set_extent_delalloc(&tmp, max_bytes, total_dirty - 1, 0, NULL); + set_extent_delalloc(tmp, max_bytes, total_dirty - 1, 0, NULL); start = test_start; end = 0; - found = find_lock_delalloc_range(inode, &tmp, locked_page, &start, + found = find_lock_delalloc_range(inode, locked_page, &start, &end); if (!found) { test_err("didn't find our range"); @@ -213,7 +215,7 @@ static int test_find_delalloc(u32 sectorsize) test_err("pages in range were not all locked"); goto out_bits; } - unlock_extent(&tmp, start, end); + unlock_extent(tmp, start, end); /* * Now to test where we run into a page that is no longer dirty in the @@ -238,7 +240,7 @@ static int test_find_delalloc(u32 sectorsize) * this changes at any point in the future we will need to fix this * tests expected behavior. */ - found = find_lock_delalloc_range(inode, &tmp, locked_page, &start, + found = find_lock_delalloc_range(inode, locked_page, &start, &end); if (!found) { test_err("didn't find our range"); @@ -256,7 +258,7 @@ static int test_find_delalloc(u32 sectorsize) } ret = 0; out_bits: - clear_extent_bits(&tmp, 0, total_dirty - 1, (unsigned)-1); + clear_extent_bits(tmp, 0, total_dirty - 1, (unsigned)-1); out: if (locked_page) put_page(locked_page); @@ -432,6 +434,89 @@ out: return ret; } +static int test_find_first_clear_extent_bit(void) +{ + struct extent_io_tree tree; + u64 start, end; + + test_msg("running find_first_clear_extent_bit test"); + extent_io_tree_init(NULL, &tree, IO_TREE_SELFTEST, NULL); + + /* + * Set 1M-4M alloc/discard and 32M-64M thus leaving a hole between + * 4M-32M + */ + set_extent_bits(&tree, SZ_1M, SZ_4M - 1, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + find_first_clear_extent_bit(&tree, SZ_512K, &start, &end, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + if (start != 0 || end != SZ_1M -1) + test_err("error finding beginning range: start %llu end %llu", + start, end); + + /* Now add 32M-64M so that we have a hole between 4M-32M */ + set_extent_bits(&tree, SZ_32M, SZ_64M - 1, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + /* + * Request first hole starting at 12M, we should get 4M-32M + */ + find_first_clear_extent_bit(&tree, 12 * SZ_1M, &start, &end, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + if (start != SZ_4M || end != SZ_32M - 1) + test_err("error finding trimmed range: start %llu end %llu", + start, end); + + /* + * Search in the middle of allocated range, should get the next one + * available, which happens to be unallocated -> 4M-32M + */ + find_first_clear_extent_bit(&tree, SZ_2M, &start, &end, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + if (start != SZ_4M || end != SZ_32M -1) + test_err("error finding next unalloc range: start %llu end %llu", + start, end); + + /* + * Set 64M-72M with CHUNK_ALLOC flag, then search for CHUNK_TRIMMED flag + * being unset in this range, we should get the entry in range 64M-72M + */ + set_extent_bits(&tree, SZ_64M, SZ_64M + SZ_8M - 1, CHUNK_ALLOCATED); + find_first_clear_extent_bit(&tree, SZ_64M + SZ_1M, &start, &end, + CHUNK_TRIMMED); + + if (start != SZ_64M || end != SZ_64M + SZ_8M - 1) + test_err("error finding exact range: start %llu end %llu", + start, end); + + find_first_clear_extent_bit(&tree, SZ_64M - SZ_8M, &start, &end, + CHUNK_TRIMMED); + + /* + * Search in the middle of set range whose immediate neighbour doesn't + * have the bits set so it must be returned + */ + if (start != SZ_64M || end != SZ_64M + SZ_8M - 1) + test_err("error finding next alloc range: start %llu end %llu", + start, end); + + /* + * Search beyond any known range, shall return after last known range + * and end should be -1 + */ + find_first_clear_extent_bit(&tree, -1, &start, &end, CHUNK_TRIMMED); + if (start != SZ_64M + SZ_8M || end != -1) + test_err( + "error handling beyond end of range search: start %llu end %llu", + start, end); + + return 0; +} + int btrfs_test_extent_io(u32 sectorsize, u32 nodesize) { int ret; @@ -442,6 +527,10 @@ int btrfs_test_extent_io(u32 sectorsize, u32 nodesize) if (ret) goto out; + ret = test_find_first_clear_extent_bit(); + if (ret) + goto out; + ret = test_eb_bitmaps(sectorsize, nodesize); out: return ret; diff --git a/fs/btrfs/tests/extent-map-tests.c b/fs/btrfs/tests/extent-map-tests.c index 87aeabe9d610..4a7f796c9900 100644 --- a/fs/btrfs/tests/extent-map-tests.c +++ b/fs/btrfs/tests/extent-map-tests.c @@ -66,7 +66,9 @@ static int test_case_1(struct btrfs_fs_info *fs_info, em->len = SZ_16K; em->block_start = 0; em->block_len = SZ_16K; + write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); + write_unlock(&em_tree->lock); if (ret < 0) { test_err("cannot add extent range [0, 16K)"); goto out; @@ -85,7 +87,9 @@ static int test_case_1(struct btrfs_fs_info *fs_info, em->len = SZ_4K; em->block_start = SZ_32K; /* avoid merging */ em->block_len = SZ_4K; + write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); + write_unlock(&em_tree->lock); if (ret < 0) { test_err("cannot add extent range [16K, 20K)"); goto out; @@ -104,7 +108,9 @@ static int test_case_1(struct btrfs_fs_info *fs_info, em->len = len; em->block_start = start; em->block_len = len; + write_lock(&em_tree->lock); ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, em->start, em->len); + write_unlock(&em_tree->lock); if (ret) { test_err("case1 [%llu %llu]: ret %d", start, start + len, ret); goto out; @@ -148,7 +154,9 @@ static int test_case_2(struct btrfs_fs_info *fs_info, em->len = SZ_1K; em->block_start = EXTENT_MAP_INLINE; em->block_len = (u64)-1; + write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); + write_unlock(&em_tree->lock); if (ret < 0) { test_err("cannot add extent range [0, 1K)"); goto out; @@ -167,7 +175,9 @@ static int test_case_2(struct btrfs_fs_info *fs_info, em->len = SZ_4K; em->block_start = SZ_4K; em->block_len = SZ_4K; + write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); + write_unlock(&em_tree->lock); if (ret < 0) { test_err("cannot add extent range [4K, 8K)"); goto out; @@ -186,7 +196,9 @@ static int test_case_2(struct btrfs_fs_info *fs_info, em->len = SZ_1K; em->block_start = EXTENT_MAP_INLINE; em->block_len = (u64)-1; + write_lock(&em_tree->lock); ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, em->start, em->len); + write_unlock(&em_tree->lock); if (ret) { test_err("case2 [0 1K]: ret %d", ret); goto out; @@ -225,7 +237,9 @@ static int __test_case_3(struct btrfs_fs_info *fs_info, em->len = SZ_4K; em->block_start = SZ_4K; em->block_len = SZ_4K; + write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); + write_unlock(&em_tree->lock); if (ret < 0) { test_err("cannot add extent range [4K, 8K)"); goto out; @@ -244,7 +258,9 @@ static int __test_case_3(struct btrfs_fs_info *fs_info, em->len = SZ_16K; em->block_start = 0; em->block_len = SZ_16K; + write_lock(&em_tree->lock); ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len); + write_unlock(&em_tree->lock); if (ret) { test_err("case3 [0x%llx 0x%llx): ret %d", start, start + len, ret); @@ -320,7 +336,9 @@ static int __test_case_4(struct btrfs_fs_info *fs_info, em->len = SZ_8K; em->block_start = 0; em->block_len = SZ_8K; + write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); + write_unlock(&em_tree->lock); if (ret < 0) { test_err("cannot add extent range [0, 8K)"); goto out; @@ -339,7 +357,9 @@ static int __test_case_4(struct btrfs_fs_info *fs_info, em->len = 24 * SZ_1K; em->block_start = SZ_16K; /* avoid merging */ em->block_len = 24 * SZ_1K; + write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); + write_unlock(&em_tree->lock); if (ret < 0) { test_err("cannot add extent range [8K, 32K)"); goto out; @@ -357,7 +377,9 @@ static int __test_case_4(struct btrfs_fs_info *fs_info, em->len = SZ_32K; em->block_start = 0; em->block_len = SZ_32K; + write_lock(&em_tree->lock); ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len); + write_unlock(&em_tree->lock); if (ret) { test_err("case4 [0x%llx 0x%llx): ret %d", start, len, ret); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 3f6811cdf803..3b8ae1a8f02d 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -129,6 +129,24 @@ static inline int extwriter_counter_read(struct btrfs_transaction *trans) } /* + * To be called after all the new block groups attached to the transaction + * handle have been created (btrfs_create_pending_block_groups()). + */ +void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + + if (!trans->chunk_bytes_reserved) + return; + + WARN_ON_ONCE(!list_empty(&trans->new_bgs)); + + btrfs_block_rsv_release(fs_info, &fs_info->chunk_block_rsv, + trans->chunk_bytes_reserved); + trans->chunk_bytes_reserved = 0; +} + +/* * either allocate a new transaction or hop into the existing one */ static noinline int join_transaction(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 78c446c222b7..527ea94b57d9 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -224,5 +224,6 @@ void btrfs_put_transaction(struct btrfs_transaction *transaction); void btrfs_apply_pending_changes(struct btrfs_fs_info *fs_info); void btrfs_add_dropped_root(struct btrfs_trans_handle *trans, struct btrfs_root *root); +void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans); #endif diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index 96fce4bef4e7..ccd5706199d7 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -132,6 +132,7 @@ static int check_extent_data_item(struct extent_buffer *leaf, struct btrfs_file_extent_item *fi; u32 sectorsize = fs_info->sectorsize; u32 item_size = btrfs_item_size_nr(leaf, slot); + u64 extent_end; if (!IS_ALIGNED(key->offset, sectorsize)) { file_extent_err(leaf, slot, @@ -207,6 +208,16 @@ static int check_extent_data_item(struct extent_buffer *leaf, CHECK_FE_ALIGNED(leaf, slot, fi, num_bytes, sectorsize)) return -EUCLEAN; + /* Catch extent end overflow */ + if (check_add_overflow(btrfs_file_extent_num_bytes(leaf, fi), + key->offset, &extent_end)) { + file_extent_err(leaf, slot, + "extent end overflow, have file offset %llu extent num bytes %llu", + key->offset, + btrfs_file_extent_num_bytes(leaf, fi)); + return -EUCLEAN; + } + /* * Check that no two consecutive file extent items, in the same leaf, * present ranges that overlap each other. diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 3fc8d854d7fb..6c8297bcfeb7 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3323,6 +3323,30 @@ int btrfs_free_log_root_tree(struct btrfs_trans_handle *trans, } /* + * Check if an inode was logged in the current transaction. We can't always rely + * on an inode's logged_trans value, because it's an in-memory only field and + * therefore not persisted. This means that its value is lost if the inode gets + * evicted and loaded again from disk (in which case it has a value of 0, and + * certainly it is smaller then any possible transaction ID), when that happens + * the full_sync flag is set in the inode's runtime flags, so on that case we + * assume eviction happened and ignore the logged_trans value, assuming the + * worst case, that the inode was logged before in the current transaction. + */ +static bool inode_logged(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + if (inode->logged_trans == trans->transid) + return true; + + if (inode->last_trans == trans->transid && + test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) && + !test_bit(BTRFS_FS_LOG_RECOVERING, &trans->fs_info->flags)) + return true; + + return false; +} + +/* * If both a file and directory are logged, and unlinks or renames are * mixed in, we have a few interesting corners: * @@ -3356,7 +3380,7 @@ int btrfs_del_dir_entries_in_log(struct btrfs_trans_handle *trans, int bytes_del = 0; u64 dir_ino = btrfs_ino(dir); - if (dir->logged_trans < trans->transid) + if (!inode_logged(trans, dir)) return 0; ret = join_running_log_trans(root); @@ -3460,7 +3484,7 @@ int btrfs_del_inode_ref_in_log(struct btrfs_trans_handle *trans, u64 index; int ret; - if (inode->logged_trans < trans->transid) + if (!inode_logged(trans, inode)) return 0; ret = join_running_log_trans(root); @@ -5420,9 +5444,19 @@ log_extents: } } + /* + * Don't update last_log_commit if we logged that an inode exists after + * it was loaded to memory (full_sync bit set). + * This is to prevent data loss when we do a write to the inode, then + * the inode gets evicted after all delalloc was flushed, then we log + * it exists (due to a rename for example) and then fsync it. This last + * fsync would do nothing (not logging the extents previously written). + */ spin_lock(&inode->lock); inode->logged_trans = trans->transid; - inode->last_log_commit = inode->last_sub_trans; + if (inode_only != LOG_INODE_EXISTS || + !test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags)) + inode->last_log_commit = inode->last_sub_trans; spin_unlock(&inode->lock); out_unlock: mutex_unlock(&inode->log_mutex); diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 1c2a6e4b39da..a13ddba1ebc3 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -28,6 +28,7 @@ #include "dev-replace.h" #include "sysfs.h" #include "tree-checker.h" +#include "space-info.h" const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = { [BTRFS_RAID_RAID10] = { @@ -123,12 +124,14 @@ const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = { }, }; -const char *get_raid_name(enum btrfs_raid_types type) +const char *btrfs_bg_type_to_raid_name(u64 flags) { - if (type >= BTRFS_NR_RAID_TYPES) + const int index = btrfs_bg_flags_to_raid_index(flags); + + if (index >= BTRFS_NR_RAID_TYPES) return NULL; - return btrfs_raid_array[type].raid_name; + return btrfs_raid_array[index].raid_name; } /* @@ -237,7 +240,9 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, * chunk_mutex * ----------- * protects chunks, adding or removing during allocation, trim or when a new - * device is added/removed + * device is added/removed. Additionally it also protects post_commit_list of + * individual devices, since they can be added to the transaction's + * post_commit_list only with chunk_mutex held. * * cleaner_mutex * ------------- @@ -1818,7 +1823,7 @@ static u64 find_next_chunk(struct btrfs_fs_info *fs_info) struct rb_node *n; u64 ret = 0; - em_tree = &fs_info->mapping_tree.map_tree; + em_tree = &fs_info->mapping_tree; read_lock(&em_tree->lock); n = rb_last(&em_tree->map.rb_root); if (n) { @@ -2941,7 +2946,7 @@ struct extent_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info, struct extent_map_tree *em_tree; struct extent_map *em; - em_tree = &fs_info->mapping_tree.map_tree; + em_tree = &fs_info->mapping_tree; read_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, logical, length); read_unlock(&em_tree->lock); @@ -3474,6 +3479,18 @@ static int chunk_devid_filter(struct extent_buffer *leaf, return 1; } +static u64 calc_data_stripes(u64 type, int num_stripes) +{ + const int index = btrfs_bg_flags_to_raid_index(type); + const int ncopies = btrfs_raid_array[index].ncopies; + const int nparity = btrfs_raid_array[index].nparity; + + if (nparity) + return num_stripes - nparity; + else + return num_stripes / ncopies; +} + /* [pstart, pend) */ static int chunk_drange_filter(struct extent_buffer *leaf, struct btrfs_chunk *chunk, @@ -3483,22 +3500,15 @@ static int chunk_drange_filter(struct extent_buffer *leaf, int num_stripes = btrfs_chunk_num_stripes(leaf, chunk); u64 stripe_offset; u64 stripe_length; + u64 type; int factor; int i; if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID)) return 0; - if (btrfs_chunk_type(leaf, chunk) & (BTRFS_BLOCK_GROUP_DUP | - BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10)) { - factor = num_stripes / 2; - } else if (btrfs_chunk_type(leaf, chunk) & BTRFS_BLOCK_GROUP_RAID5) { - factor = num_stripes - 1; - } else if (btrfs_chunk_type(leaf, chunk) & BTRFS_BLOCK_GROUP_RAID6) { - factor = num_stripes - 2; - } else { - factor = num_stripes; - } + type = btrfs_chunk_type(leaf, chunk); + factor = calc_data_stripes(type, num_stripes); for (i = 0; i < num_stripes; i++) { stripe = btrfs_stripe_nr(chunk, i); @@ -3921,11 +3931,9 @@ static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf, bp += ret; \ } while (0) - if (flags & BTRFS_BALANCE_ARGS_CONVERT) { - int index = btrfs_bg_flags_to_raid_index(bargs->target); - - CHECK_APPEND_1ARG("convert=%s,", get_raid_name(index)); - } + if (flags & BTRFS_BALANCE_ARGS_CONVERT) + CHECK_APPEND_1ARG("convert=%s,", + btrfs_bg_type_to_raid_name(bargs->target)); if (flags & BTRFS_BALANCE_ARGS_SOFT) CHECK_APPEND_NOARG("soft,"); @@ -4047,6 +4055,7 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, u64 num_devices; unsigned seq; bool reducing_integrity; + int i; if (btrfs_fs_closing(fs_info) || atomic_read(&fs_info->balance_pause_req) || @@ -4076,48 +4085,43 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, } num_devices = btrfs_num_devices(fs_info); + allowed = 0; + for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) + if (num_devices >= btrfs_raid_array[i].devs_min) + allowed |= btrfs_raid_array[i].bg_flag; - allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE | BTRFS_BLOCK_GROUP_DUP; - if (num_devices > 1) - allowed |= (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID1); - if (num_devices > 2) - allowed |= BTRFS_BLOCK_GROUP_RAID5; - if (num_devices > 3) - allowed |= (BTRFS_BLOCK_GROUP_RAID10 | - BTRFS_BLOCK_GROUP_RAID6); if (validate_convert_profile(&bctl->data, allowed)) { - int index = btrfs_bg_flags_to_raid_index(bctl->data.target); - btrfs_err(fs_info, "balance: invalid convert data profile %s", - get_raid_name(index)); + btrfs_bg_type_to_raid_name(bctl->data.target)); ret = -EINVAL; goto out; } if (validate_convert_profile(&bctl->meta, allowed)) { - int index = btrfs_bg_flags_to_raid_index(bctl->meta.target); - btrfs_err(fs_info, "balance: invalid convert metadata profile %s", - get_raid_name(index)); + btrfs_bg_type_to_raid_name(bctl->meta.target)); ret = -EINVAL; goto out; } if (validate_convert_profile(&bctl->sys, allowed)) { - int index = btrfs_bg_flags_to_raid_index(bctl->sys.target); - btrfs_err(fs_info, "balance: invalid convert system profile %s", - get_raid_name(index)); + btrfs_bg_type_to_raid_name(bctl->sys.target)); ret = -EINVAL; goto out; } - /* allow to reduce meta or sys integrity only if force set */ - allowed = BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 | - BTRFS_BLOCK_GROUP_RAID10 | - BTRFS_BLOCK_GROUP_RAID5 | - BTRFS_BLOCK_GROUP_RAID6; + /* + * Allow to reduce metadata or system integrity only if force set for + * profiles with redundancy (copies, parity) + */ + allowed = 0; + for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) { + if (btrfs_raid_array[i].ncopies >= 2 || + btrfs_raid_array[i].tolerated_failures >= 1) + allowed |= btrfs_raid_array[i].bg_flag; + } do { seq = read_seqbegin(&fs_info->profiles_lock); @@ -4152,12 +4156,18 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, if (btrfs_get_num_tolerated_disk_barrier_failures(meta_target) < btrfs_get_num_tolerated_disk_barrier_failures(data_target)) { - int meta_index = btrfs_bg_flags_to_raid_index(meta_target); - int data_index = btrfs_bg_flags_to_raid_index(data_target); - btrfs_warn(fs_info, "balance: metadata profile %s has lower redundancy than data profile %s", - get_raid_name(meta_index), get_raid_name(data_index)); + btrfs_bg_type_to_raid_name(meta_target), + btrfs_bg_type_to_raid_name(data_target)); + } + + if (fs_info->send_in_progress) { + btrfs_warn_rl(fs_info, +"cannot run balance while send operations are in progress (%d in progress)", + fs_info->send_in_progress); + ret = -EAGAIN; + goto out; } ret = insert_balance_item(fs_info, bctl); @@ -4949,6 +4959,8 @@ static int __btrfs_alloc_chunk(struct btrfs_trans_handle *trans, sub_stripes = btrfs_raid_array[index].sub_stripes; dev_stripes = btrfs_raid_array[index].dev_stripes; devs_max = btrfs_raid_array[index].devs_max; + if (!devs_max) + devs_max = BTRFS_MAX_DEVS(info); devs_min = btrfs_raid_array[index].devs_min; devs_increment = btrfs_raid_array[index].devs_increment; ncopies = btrfs_raid_array[index].ncopies; @@ -4957,8 +4969,6 @@ static int __btrfs_alloc_chunk(struct btrfs_trans_handle *trans, if (type & BTRFS_BLOCK_GROUP_DATA) { max_stripe_size = SZ_1G; max_chunk_size = BTRFS_MAX_DATA_CHUNK_SIZE; - if (!devs_max) - devs_max = BTRFS_MAX_DEVS(info); } else if (type & BTRFS_BLOCK_GROUP_METADATA) { /* for larger filesystems, use larger metadata chunks */ if (fs_devices->total_rw_bytes > 50ULL * SZ_1G) @@ -4966,13 +4976,9 @@ static int __btrfs_alloc_chunk(struct btrfs_trans_handle *trans, else max_stripe_size = SZ_256M; max_chunk_size = max_stripe_size; - if (!devs_max) - devs_max = BTRFS_MAX_DEVS(info); } else if (type & BTRFS_BLOCK_GROUP_SYSTEM) { max_stripe_size = SZ_32M; max_chunk_size = 2 * max_stripe_size; - if (!devs_max) - devs_max = BTRFS_MAX_DEVS_SYS_CHUNK; } else { btrfs_err(info, "invalid chunk type 0x%llx requested", type); @@ -5143,7 +5149,7 @@ static int __btrfs_alloc_chunk(struct btrfs_trans_handle *trans, em->block_len = em->len; em->orig_block_len = stripe_size; - em_tree = &info->mapping_tree.map_tree; + em_tree = &info->mapping_tree; write_lock(&em_tree->lock); ret = add_extent_mapping(em_tree, em, 0); if (ret) { @@ -5324,20 +5330,9 @@ static noinline int init_first_rw_device(struct btrfs_trans_handle *trans) static inline int btrfs_chunk_max_errors(struct map_lookup *map) { - int max_errors; + const int index = btrfs_bg_flags_to_raid_index(map->type); - if (map->type & (BTRFS_BLOCK_GROUP_RAID1 | - BTRFS_BLOCK_GROUP_RAID10 | - BTRFS_BLOCK_GROUP_RAID5 | - BTRFS_BLOCK_GROUP_DUP)) { - max_errors = 1; - } else if (map->type & BTRFS_BLOCK_GROUP_RAID6) { - max_errors = 2; - } else { - max_errors = 0; - } - - return max_errors; + return btrfs_raid_array[index].tolerated_failures; } int btrfs_chunk_readonly(struct btrfs_fs_info *fs_info, u64 chunk_offset) @@ -5378,21 +5373,16 @@ end: return readonly; } -void btrfs_mapping_init(struct btrfs_mapping_tree *tree) -{ - extent_map_tree_init(&tree->map_tree); -} - -void btrfs_mapping_tree_free(struct btrfs_mapping_tree *tree) +void btrfs_mapping_tree_free(struct extent_map_tree *tree) { struct extent_map *em; while (1) { - write_lock(&tree->map_tree.lock); - em = lookup_extent_mapping(&tree->map_tree, 0, (u64)-1); + write_lock(&tree->lock); + em = lookup_extent_mapping(tree, 0, (u64)-1); if (em) - remove_extent_mapping(&tree->map_tree, em); - write_unlock(&tree->map_tree.lock); + remove_extent_mapping(tree, em); + write_unlock(&tree->lock); if (!em) break; /* once for us */ @@ -5419,7 +5409,7 @@ int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len) return 1; map = em->map_lookup; - if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1)) + if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1_MASK)) ret = map->num_stripes; else if (map->type & BTRFS_BLOCK_GROUP_RAID10) ret = map->sub_stripes; @@ -5493,7 +5483,7 @@ static int find_live_mirror(struct btrfs_fs_info *fs_info, struct btrfs_device *srcdev; ASSERT((map->type & - (BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10))); + (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10))); if (map->type & BTRFS_BLOCK_GROUP_RAID10) num_stripes = map->sub_stripes; @@ -5682,7 +5672,7 @@ static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info, &remaining_stripes); div_u64_rem(stripe_nr_end - 1, factor, &last_stripe); last_stripe *= sub_stripes; - } else if (map->type & (BTRFS_BLOCK_GROUP_RAID1 | + } else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_DUP)) { num_stripes = map->num_stripes; } else { @@ -5926,6 +5916,102 @@ static bool need_full_stripe(enum btrfs_map_op op) return (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS); } +/* + * btrfs_get_io_geometry - calculates the geomery of a particular (address, len) + * tuple. This information is used to calculate how big a + * particular bio can get before it straddles a stripe. + * + * @fs_info - the filesystem + * @logical - address that we want to figure out the geometry of + * @len - the length of IO we are going to perform, starting at @logical + * @op - type of operation - write or read + * @io_geom - pointer used to return values + * + * Returns < 0 in case a chunk for the given logical address cannot be found, + * usually shouldn't happen unless @logical is corrupted, 0 otherwise. + */ +int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, + u64 logical, u64 len, struct btrfs_io_geometry *io_geom) +{ + struct extent_map *em; + struct map_lookup *map; + u64 offset; + u64 stripe_offset; + u64 stripe_nr; + u64 stripe_len; + u64 raid56_full_stripe_start = (u64)-1; + int data_stripes; + + ASSERT(op != BTRFS_MAP_DISCARD); + + em = btrfs_get_chunk_map(fs_info, logical, len); + if (IS_ERR(em)) + return PTR_ERR(em); + + map = em->map_lookup; + /* Offset of this logical address in the chunk */ + offset = logical - em->start; + /* Len of a stripe in a chunk */ + stripe_len = map->stripe_len; + /* Stripe wher this block falls in */ + stripe_nr = div64_u64(offset, stripe_len); + /* Offset of stripe in the chunk */ + stripe_offset = stripe_nr * stripe_len; + if (offset < stripe_offset) { + btrfs_crit(fs_info, +"stripe math has gone wrong, stripe_offset=%llu offset=%llu start=%llu logical=%llu stripe_len=%llu", + stripe_offset, offset, em->start, logical, stripe_len); + free_extent_map(em); + return -EINVAL; + } + + /* stripe_offset is the offset of this block in its stripe */ + stripe_offset = offset - stripe_offset; + data_stripes = nr_data_stripes(map); + + if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { + u64 max_len = stripe_len - stripe_offset; + + /* + * In case of raid56, we need to know the stripe aligned start + */ + if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + unsigned long full_stripe_len = stripe_len * data_stripes; + raid56_full_stripe_start = offset; + + /* + * Allow a write of a full stripe, but make sure we + * don't allow straddling of stripes + */ + raid56_full_stripe_start = div64_u64(raid56_full_stripe_start, + full_stripe_len); + raid56_full_stripe_start *= full_stripe_len; + + /* + * For writes to RAID[56], allow a full stripeset across + * all disks. For other RAID types and for RAID[56] + * reads, just allow a single stripe (on a single disk). + */ + if (op == BTRFS_MAP_WRITE) { + max_len = stripe_len * data_stripes - + (offset - raid56_full_stripe_start); + } + } + len = min_t(u64, em->len - offset, max_len); + } else { + len = em->len - offset; + } + + io_geom->len = len; + io_geom->offset = offset; + io_geom->stripe_len = stripe_len; + io_geom->stripe_nr = stripe_nr; + io_geom->stripe_offset = stripe_offset; + io_geom->raid56_stripe_offset = raid56_full_stripe_start; + + return 0; +} + static int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, u64 logical, u64 *length, @@ -5939,6 +6025,7 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, u64 stripe_nr; u64 stripe_len; u32 stripe_index; + int data_stripes; int i; int ret = 0; int num_stripes; @@ -5951,76 +6038,29 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, int patch_the_first_stripe_for_dev_replace = 0; u64 physical_to_patch_in_first_stripe = 0; u64 raid56_full_stripe_start = (u64)-1; + struct btrfs_io_geometry geom; + + ASSERT(bbio_ret); if (op == BTRFS_MAP_DISCARD) return __btrfs_map_block_for_discard(fs_info, logical, *length, bbio_ret); - em = btrfs_get_chunk_map(fs_info, logical, *length); - if (IS_ERR(em)) - return PTR_ERR(em); + ret = btrfs_get_io_geometry(fs_info, op, logical, *length, &geom); + if (ret < 0) + return ret; + em = btrfs_get_chunk_map(fs_info, logical, *length); + ASSERT(em); map = em->map_lookup; - offset = logical - em->start; - - stripe_len = map->stripe_len; - stripe_nr = offset; - /* - * stripe_nr counts the total number of stripes we have to stride - * to get to this block - */ - stripe_nr = div64_u64(stripe_nr, stripe_len); - - stripe_offset = stripe_nr * stripe_len; - if (offset < stripe_offset) { - btrfs_crit(fs_info, - "stripe math has gone wrong, stripe_offset=%llu, offset=%llu, start=%llu, logical=%llu, stripe_len=%llu", - stripe_offset, offset, em->start, logical, - stripe_len); - free_extent_map(em); - return -EINVAL; - } - - /* stripe_offset is the offset of this block in its stripe*/ - stripe_offset = offset - stripe_offset; - - /* if we're here for raid56, we need to know the stripe aligned start */ - if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { - unsigned long full_stripe_len = stripe_len * nr_data_stripes(map); - raid56_full_stripe_start = offset; - /* allow a write of a full stripe, but make sure we don't - * allow straddling of stripes - */ - raid56_full_stripe_start = div64_u64(raid56_full_stripe_start, - full_stripe_len); - raid56_full_stripe_start *= full_stripe_len; - } - - if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { - u64 max_len; - /* For writes to RAID[56], allow a full stripeset across all disks. - For other RAID types and for RAID[56] reads, just allow a single - stripe (on a single disk). */ - if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && - (op == BTRFS_MAP_WRITE)) { - max_len = stripe_len * nr_data_stripes(map) - - (offset - raid56_full_stripe_start); - } else { - /* we limit the length of each bio to what fits in a stripe */ - max_len = stripe_len - stripe_offset; - } - *length = min_t(u64, em->len - offset, max_len); - } else { - *length = em->len - offset; - } - - /* - * This is for when we're called from btrfs_bio_fits_in_stripe and all - * it cares about is the length - */ - if (!bbio_ret) - goto out; + *length = geom.len; + offset = geom.offset; + stripe_len = geom.stripe_len; + stripe_nr = geom.stripe_nr; + stripe_offset = geom.stripe_offset; + raid56_full_stripe_start = geom.raid56_stripe_offset; + data_stripes = nr_data_stripes(map); down_read(&dev_replace->rwsem); dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace); @@ -6052,7 +6092,7 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, &stripe_index); if (!need_full_stripe(op)) mirror_num = 1; - } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) { + } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) { if (need_full_stripe(op)) num_stripes = map->num_stripes; else if (mirror_num) @@ -6094,7 +6134,7 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, if (need_raid_map && (need_full_stripe(op) || mirror_num > 1)) { /* push stripe_nr back to the start of the full stripe */ stripe_nr = div64_u64(raid56_full_stripe_start, - stripe_len * nr_data_stripes(map)); + stripe_len * data_stripes); /* RAID[56] write or recovery. Return all stripes */ num_stripes = map->num_stripes; @@ -6110,10 +6150,9 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, * Mirror #3 is RAID6 Q block. */ stripe_nr = div_u64_rem(stripe_nr, - nr_data_stripes(map), &stripe_index); + data_stripes, &stripe_index); if (mirror_num > 1) - stripe_index = nr_data_stripes(map) + - mirror_num - 2; + stripe_index = data_stripes + mirror_num - 2; /* We distribute the parity blocks across stripes */ div_u64_rem(stripe_nr + stripe_index, map->num_stripes, @@ -6171,8 +6210,8 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, div_u64_rem(stripe_nr, num_stripes, &rot); /* Fill in the logical address of each stripe */ - tmp = stripe_nr * nr_data_stripes(map); - for (i = 0; i < nr_data_stripes(map); i++) + tmp = stripe_nr * data_stripes; + for (i = 0; i < data_stripes; i++) bbio->raid_map[(i+rot) % num_stripes] = em->start + (tmp + i) * map->stripe_len; @@ -6687,7 +6726,7 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, struct btrfs_chunk *chunk) { struct btrfs_fs_info *fs_info = leaf->fs_info; - struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree; + struct extent_map_tree *map_tree = &fs_info->mapping_tree; struct map_lookup *map; struct extent_map *em; u64 logical; @@ -6712,9 +6751,9 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, return ret; } - read_lock(&map_tree->map_tree.lock); - em = lookup_extent_mapping(&map_tree->map_tree, logical, 1); - read_unlock(&map_tree->map_tree.lock); + read_lock(&map_tree->lock); + em = lookup_extent_mapping(map_tree, logical, 1); + read_unlock(&map_tree->lock); /* already mapped? */ if (em && em->start <= logical && em->start + em->len > logical) { @@ -6783,9 +6822,9 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, } - write_lock(&map_tree->map_tree.lock); - ret = add_extent_mapping(&map_tree->map_tree, em, 0); - write_unlock(&map_tree->map_tree.lock); + write_lock(&map_tree->lock); + ret = add_extent_mapping(map_tree, em, 0); + write_unlock(&map_tree->lock); if (ret < 0) { btrfs_err(fs_info, "failed to add chunk map, start=%llu len=%llu: %d", @@ -7103,14 +7142,14 @@ out_short_read: bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info, struct btrfs_device *failing_dev) { - struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree; + struct extent_map_tree *map_tree = &fs_info->mapping_tree; struct extent_map *em; u64 next_start = 0; bool ret = true; - read_lock(&map_tree->map_tree.lock); - em = lookup_extent_mapping(&map_tree->map_tree, 0, (u64)-1); - read_unlock(&map_tree->map_tree.lock); + read_lock(&map_tree->lock); + em = lookup_extent_mapping(map_tree, 0, (u64)-1); + read_unlock(&map_tree->lock); /* No chunk at all? Return false anyway */ if (!em) { ret = false; @@ -7148,10 +7187,10 @@ bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info, next_start = extent_map_end(em); free_extent_map(em); - read_lock(&map_tree->map_tree.lock); - em = lookup_extent_mapping(&map_tree->map_tree, next_start, + read_lock(&map_tree->lock); + em = lookup_extent_mapping(map_tree, next_start, (u64)(-1) - next_start); - read_unlock(&map_tree->map_tree.lock); + read_unlock(&map_tree->lock); } out: return ret; @@ -7600,10 +7639,9 @@ void btrfs_reset_fs_info_ptr(struct btrfs_fs_info *fs_info) */ int btrfs_bg_type_to_factor(u64 flags) { - if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 | - BTRFS_BLOCK_GROUP_RAID10)) - return 2; - return 1; + const int index = btrfs_bg_flags_to_raid_index(flags); + + return btrfs_raid_array[index].ncopies; } @@ -7612,7 +7650,7 @@ static int verify_one_dev_extent(struct btrfs_fs_info *fs_info, u64 chunk_offset, u64 devid, u64 physical_offset, u64 physical_len) { - struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree; + struct extent_map_tree *em_tree = &fs_info->mapping_tree; struct extent_map *em; struct map_lookup *map; struct btrfs_device *dev; @@ -7701,7 +7739,7 @@ out: static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info) { - struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree; + struct extent_map_tree *em_tree = &fs_info->mapping_tree; struct extent_map *em; struct rb_node *node; int ret = 0; diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 136a3eb64604..7f6aa1816409 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -23,6 +23,21 @@ struct btrfs_pending_bios { struct bio *tail; }; +struct btrfs_io_geometry { + /* remaining bytes before crossing a stripe */ + u64 len; + /* offset of logical address in chunk */ + u64 offset; + /* length of single IO stripe */ + u64 stripe_len; + /* number of stripe where address falls */ + u64 stripe_nr; + /* offset of address in stripe */ + u64 stripe_offset; + /* offset of raid56 stripe into the chunk */ + u64 raid56_stripe_offset; +}; + /* * Use sequence counter to get consistent device stat data on * 32-bit processors. @@ -43,8 +58,8 @@ struct btrfs_pending_bios { #define BTRFS_DEV_STATE_FLUSH_SENT (4) struct btrfs_device { - struct list_head dev_list; - struct list_head dev_alloc_list; + struct list_head dev_list; /* device_list_mutex */ + struct list_head dev_alloc_list; /* chunk mutex */ struct list_head post_commit_list; /* chunk mutex */ struct btrfs_fs_devices *fs_devices; struct btrfs_fs_info *fs_info; @@ -229,9 +244,14 @@ struct btrfs_fs_devices { * this mutex lock. */ struct mutex device_list_mutex; + + /* List of all devices, protected by device_list_mutex */ struct list_head devices; - /* devices not currently being allocated */ + /* + * Devices which can satisfy space allocation. Protected by + * chunk_mutex + */ struct list_head alloc_list; struct btrfs_fs_devices *seed; @@ -336,16 +356,16 @@ struct btrfs_device_info { }; struct btrfs_raid_attr { - int sub_stripes; /* sub_stripes info for map */ - int dev_stripes; /* stripes per dev */ - int devs_max; /* max devs to use */ - int devs_min; /* min devs needed */ - int tolerated_failures; /* max tolerated fail devs */ - int devs_increment; /* ndevs has to be a multiple of this */ - int ncopies; /* how many copies to data has */ - int nparity; /* number of stripes worth of bytes to store + u8 sub_stripes; /* sub_stripes info for map */ + u8 dev_stripes; /* stripes per dev */ + u8 devs_max; /* max devs to use */ + u8 devs_min; /* min devs needed */ + u8 tolerated_failures; /* max tolerated fail devs */ + u8 devs_increment; /* ndevs has to be a multiple of this */ + u8 ncopies; /* how many copies to data has */ + u8 nparity; /* number of stripes worth of bytes to store * parity information */ - int mindev_error; /* error code if min devs requisite is unmet */ + u8 mindev_error; /* error code if min devs requisite is unmet */ const char raid_name[8]; /* name of the raid */ u64 bg_flag; /* block group flag of the raid */ }; @@ -408,13 +428,14 @@ int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, u64 logical, u64 *length, struct btrfs_bio **bbio_ret); +int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, + u64 logical, u64 len, struct btrfs_io_geometry *io_geom); int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, u64 physical, u64 **logical, int *naddrs, int *stripe_len); int btrfs_read_sys_array(struct btrfs_fs_info *fs_info); int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info); int btrfs_alloc_chunk(struct btrfs_trans_handle *trans, u64 type); -void btrfs_mapping_init(struct btrfs_mapping_tree *tree); -void btrfs_mapping_tree_free(struct btrfs_mapping_tree *tree); +void btrfs_mapping_tree_free(struct extent_map_tree *tree); blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num, int async_submit); int btrfs_open_devices(struct btrfs_fs_devices *fs_devices, @@ -557,8 +578,6 @@ static inline enum btrfs_raid_types btrfs_bg_flags_to_raid_index(u64 flags) return BTRFS_RAID_SINGLE; /* BTRFS_BLOCK_GROUP_SINGLE */ } -const char *get_raid_name(enum btrfs_raid_types type); - void btrfs_commit_device_sizes(struct btrfs_transaction *trans); struct list_head *btrfs_get_fs_uuids(void); @@ -568,6 +587,7 @@ bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info, struct btrfs_device *failing_dev); int btrfs_bg_type_to_factor(u64 flags); +const char *btrfs_bg_type_to_raid_name(u64 flags); int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info); #endif diff --git a/fs/ceph/Kconfig b/fs/ceph/Kconfig index 7f7d92d6b024..cf235f6eacf9 100644 --- a/fs/ceph/Kconfig +++ b/fs/ceph/Kconfig @@ -36,3 +36,15 @@ config CEPH_FS_POSIX_ACL groups beyond the owner/group/world scheme. If you don't know what Access Control Lists are, say N + +config CEPH_FS_SECURITY_LABEL + bool "CephFS Security Labels" + depends on CEPH_FS && SECURITY + help + Security labels support alternative access control models + implemented by security modules like SELinux. This option + enables an extended attribute handler for file security + labels in the Ceph filesystem. + + If you are not using a security module that requires using + extended attributes for file security labels, say N. diff --git a/fs/ceph/acl.c b/fs/ceph/acl.c index 8a19c249036c..aa55f412a6e3 100644 --- a/fs/ceph/acl.c +++ b/fs/ceph/acl.c @@ -159,7 +159,7 @@ out: } int ceph_pre_init_acls(struct inode *dir, umode_t *mode, - struct ceph_acls_info *info) + struct ceph_acl_sec_ctx *as_ctx) { struct posix_acl *acl, *default_acl; size_t val_size1 = 0, val_size2 = 0; @@ -234,9 +234,9 @@ int ceph_pre_init_acls(struct inode *dir, umode_t *mode, kfree(tmp_buf); - info->acl = acl; - info->default_acl = default_acl; - info->pagelist = pagelist; + as_ctx->acl = acl; + as_ctx->default_acl = default_acl; + as_ctx->pagelist = pagelist; return 0; out_err: @@ -248,18 +248,10 @@ out_err: return err; } -void ceph_init_inode_acls(struct inode* inode, struct ceph_acls_info *info) +void ceph_init_inode_acls(struct inode *inode, struct ceph_acl_sec_ctx *as_ctx) { if (!inode) return; - ceph_set_cached_acl(inode, ACL_TYPE_ACCESS, info->acl); - ceph_set_cached_acl(inode, ACL_TYPE_DEFAULT, info->default_acl); -} - -void ceph_release_acls_info(struct ceph_acls_info *info) -{ - posix_acl_release(info->acl); - posix_acl_release(info->default_acl); - if (info->pagelist) - ceph_pagelist_release(info->pagelist); + ceph_set_cached_acl(inode, ACL_TYPE_ACCESS, as_ctx->acl); + ceph_set_cached_acl(inode, ACL_TYPE_DEFAULT, as_ctx->default_acl); } diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index a47c541f8006..e078cc55b989 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -10,6 +10,7 @@ #include <linux/pagevec.h> #include <linux/task_io_accounting_ops.h> #include <linux/signal.h> +#include <linux/iversion.h> #include "super.h" #include "mds_client.h" @@ -1576,6 +1577,7 @@ static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf) /* Update time before taking page lock */ file_update_time(vma->vm_file); + inode_inc_iversion_raw(inode); do { lock_page(page); diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 0176241eaea7..d98dcd976c80 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -8,6 +8,7 @@ #include <linux/vmalloc.h> #include <linux/wait.h> #include <linux/writeback.h> +#include <linux/iversion.h> #include "super.h" #include "mds_client.h" @@ -1138,8 +1139,9 @@ struct cap_msg_args { u64 ino, cid, follows; u64 flush_tid, oldest_flush_tid, size, max_size; u64 xattr_version; + u64 change_attr; struct ceph_buffer *xattr_buf; - struct timespec64 atime, mtime, ctime; + struct timespec64 atime, mtime, ctime, btime; int op, caps, wanted, dirty; u32 seq, issue_seq, mseq, time_warp_seq; u32 flags; @@ -1160,7 +1162,6 @@ static int send_cap_msg(struct cap_msg_args *arg) struct ceph_msg *msg; void *p; size_t extra_len; - struct timespec64 zerotime = {0}; struct ceph_osd_client *osdc = &arg->session->s_mdsc->fsc->client->osdc; dout("send_cap_msg %s %llx %llx caps %s wanted %s dirty %s" @@ -1245,15 +1246,10 @@ static int send_cap_msg(struct cap_msg_args *arg) /* pool namespace (version 8) (mds always ignores this) */ ceph_encode_32(&p, 0); - /* - * btime and change_attr (version 9) - * - * We just zero these out for now, as the MDS ignores them unless - * the requisite feature flags are set (which we don't do yet). - */ - ceph_encode_timespec64(p, &zerotime); + /* btime and change_attr (version 9) */ + ceph_encode_timespec64(p, &arg->btime); p += sizeof(struct ceph_timespec); - ceph_encode_64(&p, 0); + ceph_encode_64(&p, arg->change_attr); /* Advisory flags (version 10) */ ceph_encode_32(&p, arg->flags); @@ -1263,20 +1259,22 @@ static int send_cap_msg(struct cap_msg_args *arg) } /* - * Queue cap releases when an inode is dropped from our cache. Since - * inode is about to be destroyed, there is no need for i_ceph_lock. + * Queue cap releases when an inode is dropped from our cache. */ -void __ceph_remove_caps(struct inode *inode) +void __ceph_remove_caps(struct ceph_inode_info *ci) { - struct ceph_inode_info *ci = ceph_inode(inode); struct rb_node *p; + /* lock i_ceph_lock, because ceph_d_revalidate(..., LOOKUP_RCU) + * may call __ceph_caps_issued_mask() on a freeing inode. */ + spin_lock(&ci->i_ceph_lock); p = rb_first(&ci->i_caps); while (p) { struct ceph_cap *cap = rb_entry(p, struct ceph_cap, ci_node); p = rb_next(p); __ceph_remove_cap(cap, true); } + spin_unlock(&ci->i_ceph_lock); } /* @@ -1297,7 +1295,7 @@ void __ceph_remove_caps(struct inode *inode) * caller should hold snap_rwsem (read), s_mutex. */ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, - int op, bool sync, int used, int want, int retain, + int op, int flags, int used, int want, int retain, int flushing, u64 flush_tid, u64 oldest_flush_tid) __releases(cap->ci->i_ceph_lock) { @@ -1377,6 +1375,8 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, arg.mtime = inode->i_mtime; arg.atime = inode->i_atime; arg.ctime = inode->i_ctime; + arg.btime = ci->i_btime; + arg.change_attr = inode_peek_iversion_raw(inode); arg.op = op; arg.caps = cap->implemented; @@ -1393,12 +1393,19 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, arg.mode = inode->i_mode; arg.inline_data = ci->i_inline_version != CEPH_INLINE_NONE; - if (list_empty(&ci->i_cap_snaps)) - arg.flags = CEPH_CLIENT_CAPS_NO_CAPSNAP; - else - arg.flags = CEPH_CLIENT_CAPS_PENDING_CAPSNAP; - if (sync) - arg.flags |= CEPH_CLIENT_CAPS_SYNC; + if (!(flags & CEPH_CLIENT_CAPS_PENDING_CAPSNAP) && + !list_empty(&ci->i_cap_snaps)) { + struct ceph_cap_snap *capsnap; + list_for_each_entry_reverse(capsnap, &ci->i_cap_snaps, ci_item) { + if (capsnap->cap_flush.tid) + break; + if (capsnap->need_flush) { + flags |= CEPH_CLIENT_CAPS_PENDING_CAPSNAP; + break; + } + } + } + arg.flags = flags; spin_unlock(&ci->i_ceph_lock); @@ -1436,6 +1443,8 @@ static inline int __send_flush_snap(struct inode *inode, arg.atime = capsnap->atime; arg.mtime = capsnap->mtime; arg.ctime = capsnap->ctime; + arg.btime = capsnap->btime; + arg.change_attr = capsnap->change_attr; arg.op = CEPH_CAP_OP_FLUSHSNAP; arg.caps = capsnap->issued; @@ -1603,10 +1612,8 @@ retry: } // make sure flushsnap messages are sent in proper order. - if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) { + if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) __kick_flushing_caps(mdsc, session, ci, 0); - ci->i_ceph_flags &= ~CEPH_I_KICK_FLUSH; - } __ceph_flush_snaps(ci, session); out: @@ -2048,10 +2055,8 @@ ack: if (cap == ci->i_auth_cap && (ci->i_ceph_flags & (CEPH_I_KICK_FLUSH | CEPH_I_FLUSH_SNAPS))) { - if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) { + if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) __kick_flushing_caps(mdsc, session, ci, 0); - ci->i_ceph_flags &= ~CEPH_I_KICK_FLUSH; - } if (ci->i_ceph_flags & CEPH_I_FLUSH_SNAPS) __ceph_flush_snaps(ci, session); @@ -2087,7 +2092,7 @@ ack: sent++; /* __send_cap drops i_ceph_lock */ - delayed += __send_cap(mdsc, cap, CEPH_CAP_OP_UPDATE, false, + delayed += __send_cap(mdsc, cap, CEPH_CAP_OP_UPDATE, 0, cap_used, want, retain, flushing, flush_tid, oldest_flush_tid); goto retry; /* retake i_ceph_lock and restart our cap scan. */ @@ -2121,6 +2126,7 @@ static int try_flush_caps(struct inode *inode, u64 *ptid) retry: spin_lock(&ci->i_ceph_lock); +retry_locked: if (ci->i_ceph_flags & CEPH_I_NOFLUSH) { spin_unlock(&ci->i_ceph_lock); dout("try_flush_caps skipping %p I_NOFLUSH set\n", inode); @@ -2128,8 +2134,6 @@ retry: } if (ci->i_dirty_caps && ci->i_auth_cap) { struct ceph_cap *cap = ci->i_auth_cap; - int used = __ceph_caps_used(ci); - int want = __ceph_caps_wanted(ci); int delayed; if (!session || session != cap->session) { @@ -2145,13 +2149,25 @@ retry: goto out; } + if (ci->i_ceph_flags & + (CEPH_I_KICK_FLUSH | CEPH_I_FLUSH_SNAPS)) { + if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) + __kick_flushing_caps(mdsc, session, ci, 0); + if (ci->i_ceph_flags & CEPH_I_FLUSH_SNAPS) + __ceph_flush_snaps(ci, session); + goto retry_locked; + } + flushing = __mark_caps_flushing(inode, session, true, &flush_tid, &oldest_flush_tid); /* __send_cap drops i_ceph_lock */ - delayed = __send_cap(mdsc, cap, CEPH_CAP_OP_FLUSH, true, - used, want, (cap->issued | cap->implemented), - flushing, flush_tid, oldest_flush_tid); + delayed = __send_cap(mdsc, cap, CEPH_CAP_OP_FLUSH, + CEPH_CLIENT_CAPS_SYNC, + __ceph_caps_used(ci), + __ceph_caps_wanted(ci), + (cap->issued | cap->implemented), + flushing, flush_tid, oldest_flush_tid); if (delayed) { spin_lock(&ci->i_ceph_lock); @@ -2320,6 +2336,16 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, struct ceph_cap_flush *cf; int ret; u64 first_tid = 0; + u64 last_snap_flush = 0; + + ci->i_ceph_flags &= ~CEPH_I_KICK_FLUSH; + + list_for_each_entry_reverse(cf, &ci->i_cap_flush_list, i_list) { + if (!cf->caps) { + last_snap_flush = cf->tid; + break; + } + } list_for_each_entry(cf, &ci->i_cap_flush_list, i_list) { if (cf->tid < first_tid) @@ -2338,10 +2364,13 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, dout("kick_flushing_caps %p cap %p tid %llu %s\n", inode, cap, cf->tid, ceph_cap_string(cf->caps)); ci->i_ceph_flags |= CEPH_I_NODELAY; + ret = __send_cap(mdsc, cap, CEPH_CAP_OP_FLUSH, - false, __ceph_caps_used(ci), + (cf->tid < last_snap_flush ? + CEPH_CLIENT_CAPS_PENDING_CAPSNAP : 0), + __ceph_caps_used(ci), __ceph_caps_wanted(ci), - cap->issued | cap->implemented, + (cap->issued | cap->implemented), cf->caps, cf->tid, oldest_flush_tid); if (ret) { pr_err("kick_flushing_caps: error sending " @@ -2410,7 +2439,6 @@ void ceph_early_kick_flushing_caps(struct ceph_mds_client *mdsc, */ if ((cap->issued & ci->i_flushing_caps) != ci->i_flushing_caps) { - ci->i_ceph_flags &= ~CEPH_I_KICK_FLUSH; /* encode_caps_cb() also will reset these sequence * numbers. make sure sequence numbers in cap flush * message match later reconnect message */ @@ -2450,7 +2478,6 @@ void ceph_kick_flushing_caps(struct ceph_mds_client *mdsc, continue; } if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) { - ci->i_ceph_flags &= ~CEPH_I_KICK_FLUSH; __kick_flushing_caps(mdsc, session, ci, oldest_flush_tid); } @@ -2478,7 +2505,6 @@ static void kick_flushing_inode_caps(struct ceph_mds_client *mdsc, oldest_flush_tid = __get_oldest_flush_tid(mdsc); spin_unlock(&mdsc->cap_dirty_lock); - ci->i_ceph_flags &= ~CEPH_I_KICK_FLUSH; __kick_flushing_caps(mdsc, session, ci, oldest_flush_tid); spin_unlock(&ci->i_ceph_lock); } else { @@ -3040,8 +3066,10 @@ struct cap_extra_info { bool dirstat_valid; u64 nfiles; u64 nsubdirs; + u64 change_attr; /* currently issued */ int issued; + struct timespec64 btime; }; /* @@ -3123,11 +3151,14 @@ static void handle_cap_grant(struct inode *inode, __check_cap_issue(ci, cap, newcaps); + inode_set_max_iversion_raw(inode, extra_info->change_attr); + if ((newcaps & CEPH_CAP_AUTH_SHARED) && (extra_info->issued & CEPH_CAP_AUTH_EXCL) == 0) { inode->i_mode = le32_to_cpu(grant->mode); inode->i_uid = make_kuid(&init_user_ns, le32_to_cpu(grant->uid)); inode->i_gid = make_kgid(&init_user_ns, le32_to_cpu(grant->gid)); + ci->i_btime = extra_info->btime; dout("%p mode 0%o uid.gid %d.%d\n", inode, inode->i_mode, from_kuid(&init_user_ns, inode->i_uid), from_kgid(&init_user_ns, inode->i_gid)); @@ -3154,6 +3185,7 @@ static void handle_cap_grant(struct inode *inode, ci->i_xattrs.blob = ceph_buffer_get(xattr_buf); ci->i_xattrs.version = version; ceph_forget_all_cached_acls(inode); + ceph_security_invalidate_secctx(inode); } } @@ -3848,17 +3880,19 @@ void ceph_handle_caps(struct ceph_mds_session *session, } } - if (msg_version >= 11) { + if (msg_version >= 9) { struct ceph_timespec *btime; - u64 change_attr; - u32 flags; - /* version >= 9 */ if (p + sizeof(*btime) > end) goto bad; btime = p; + ceph_decode_timespec64(&extra_info.btime, btime); p += sizeof(*btime); - ceph_decode_64_safe(&p, end, change_attr, bad); + ceph_decode_64_safe(&p, end, extra_info.change_attr, bad); + } + + if (msg_version >= 11) { + u32 flags; /* version >= 10 */ ceph_decode_32_safe(&p, end, flags, bad); /* version >= 11 */ diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c index 83cd41fa2b01..2eb88ed22993 100644 --- a/fs/ceph/debugfs.c +++ b/fs/ceph/debugfs.c @@ -52,7 +52,7 @@ static int mdsc_show(struct seq_file *s, void *p) struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_mds_request *req; struct rb_node *rp; - int pathlen; + int pathlen = 0; u64 pathbase; char *path; diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 0637149fb9f9..aab29f48c62d 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -825,7 +825,7 @@ static int ceph_mknod(struct inode *dir, struct dentry *dentry, struct ceph_fs_client *fsc = ceph_sb_to_client(dir->i_sb); struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_mds_request *req; - struct ceph_acls_info acls = {}; + struct ceph_acl_sec_ctx as_ctx = {}; int err; if (ceph_snap(dir) != CEPH_NOSNAP) @@ -836,7 +836,10 @@ static int ceph_mknod(struct inode *dir, struct dentry *dentry, goto out; } - err = ceph_pre_init_acls(dir, &mode, &acls); + err = ceph_pre_init_acls(dir, &mode, &as_ctx); + if (err < 0) + goto out; + err = ceph_security_init_secctx(dentry, mode, &as_ctx); if (err < 0) goto out; @@ -855,9 +858,9 @@ static int ceph_mknod(struct inode *dir, struct dentry *dentry, req->r_args.mknod.rdev = cpu_to_le32(rdev); req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL; req->r_dentry_unless = CEPH_CAP_FILE_EXCL; - if (acls.pagelist) { - req->r_pagelist = acls.pagelist; - acls.pagelist = NULL; + if (as_ctx.pagelist) { + req->r_pagelist = as_ctx.pagelist; + as_ctx.pagelist = NULL; } err = ceph_mdsc_do_request(mdsc, dir, req); if (!err && !req->r_reply_info.head->is_dentry) @@ -865,10 +868,10 @@ static int ceph_mknod(struct inode *dir, struct dentry *dentry, ceph_mdsc_put_request(req); out: if (!err) - ceph_init_inode_acls(d_inode(dentry), &acls); + ceph_init_inode_acls(d_inode(dentry), &as_ctx); else d_drop(dentry); - ceph_release_acls_info(&acls); + ceph_release_acl_sec_ctx(&as_ctx); return err; } @@ -884,6 +887,7 @@ static int ceph_symlink(struct inode *dir, struct dentry *dentry, struct ceph_fs_client *fsc = ceph_sb_to_client(dir->i_sb); struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_mds_request *req; + struct ceph_acl_sec_ctx as_ctx = {}; int err; if (ceph_snap(dir) != CEPH_NOSNAP) @@ -894,6 +898,10 @@ static int ceph_symlink(struct inode *dir, struct dentry *dentry, goto out; } + err = ceph_security_init_secctx(dentry, S_IFLNK | 0777, &as_ctx); + if (err < 0) + goto out; + dout("symlink in dir %p dentry %p to '%s'\n", dir, dentry, dest); req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SYMLINK, USE_AUTH_MDS); if (IS_ERR(req)) { @@ -919,6 +927,7 @@ static int ceph_symlink(struct inode *dir, struct dentry *dentry, out: if (err) d_drop(dentry); + ceph_release_acl_sec_ctx(&as_ctx); return err; } @@ -927,7 +936,7 @@ static int ceph_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) struct ceph_fs_client *fsc = ceph_sb_to_client(dir->i_sb); struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_mds_request *req; - struct ceph_acls_info acls = {}; + struct ceph_acl_sec_ctx as_ctx = {}; int err = -EROFS; int op; @@ -950,7 +959,10 @@ static int ceph_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) } mode |= S_IFDIR; - err = ceph_pre_init_acls(dir, &mode, &acls); + err = ceph_pre_init_acls(dir, &mode, &as_ctx); + if (err < 0) + goto out; + err = ceph_security_init_secctx(dentry, mode, &as_ctx); if (err < 0) goto out; @@ -967,9 +979,9 @@ static int ceph_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) req->r_args.mkdir.mode = cpu_to_le32(mode); req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL; req->r_dentry_unless = CEPH_CAP_FILE_EXCL; - if (acls.pagelist) { - req->r_pagelist = acls.pagelist; - acls.pagelist = NULL; + if (as_ctx.pagelist) { + req->r_pagelist = as_ctx.pagelist; + as_ctx.pagelist = NULL; } err = ceph_mdsc_do_request(mdsc, dir, req); if (!err && @@ -979,10 +991,10 @@ static int ceph_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) ceph_mdsc_put_request(req); out: if (!err) - ceph_init_inode_acls(d_inode(dentry), &acls); + ceph_init_inode_acls(d_inode(dentry), &as_ctx); else d_drop(dentry); - ceph_release_acls_info(&acls); + ceph_release_acl_sec_ctx(&as_ctx); return err; } @@ -1433,8 +1445,7 @@ static bool __dentry_lease_is_valid(struct ceph_dentry_info *di) return false; } -static int dentry_lease_is_valid(struct dentry *dentry, unsigned int flags, - struct inode *dir) +static int dentry_lease_is_valid(struct dentry *dentry, unsigned int flags) { struct ceph_dentry_info *di; struct ceph_mds_session *session = NULL; @@ -1466,7 +1477,7 @@ static int dentry_lease_is_valid(struct dentry *dentry, unsigned int flags, spin_unlock(&dentry->d_lock); if (session) { - ceph_mdsc_lease_send_msg(session, dir, dentry, + ceph_mdsc_lease_send_msg(session, dentry, CEPH_MDS_LEASE_RENEW, seq); ceph_put_mds_session(session); } @@ -1512,18 +1523,26 @@ static int __dir_lease_try_check(const struct dentry *dentry) static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry) { struct ceph_inode_info *ci = ceph_inode(dir); - struct ceph_dentry_info *di = ceph_dentry(dentry); - int valid = 0; + int valid; + int shared_gen; spin_lock(&ci->i_ceph_lock); - if (atomic_read(&ci->i_shared_gen) == di->lease_shared_gen) - valid = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1); + valid = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1); + shared_gen = atomic_read(&ci->i_shared_gen); spin_unlock(&ci->i_ceph_lock); - if (valid) - __ceph_dentry_dir_lease_touch(di); - dout("dir_lease_is_valid dir %p v%u dentry %p v%u = %d\n", - dir, (unsigned)atomic_read(&ci->i_shared_gen), - dentry, (unsigned)di->lease_shared_gen, valid); + if (valid) { + struct ceph_dentry_info *di; + spin_lock(&dentry->d_lock); + di = ceph_dentry(dentry); + if (dir == d_inode(dentry->d_parent) && + di && di->lease_shared_gen == shared_gen) + __ceph_dentry_dir_lease_touch(di); + else + valid = 0; + spin_unlock(&dentry->d_lock); + } + dout("dir_lease_is_valid dir %p v%u dentry %p = %d\n", + dir, (unsigned)atomic_read(&ci->i_shared_gen), dentry, valid); return valid; } @@ -1558,7 +1577,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) ceph_snap(d_inode(dentry)) == CEPH_SNAPDIR) { valid = 1; } else { - valid = dentry_lease_is_valid(dentry, flags, dir); + valid = dentry_lease_is_valid(dentry, flags); if (valid == -ECHILD) return valid; if (valid || dir_lease_is_valid(dir, dentry)) { diff --git a/fs/ceph/export.c b/fs/ceph/export.c index d3ef7ee429ec..15ff1b09cfa2 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -368,7 +368,7 @@ static struct dentry *ceph_get_parent(struct dentry *child) } out: dout("get_parent %p ino %llx.%llx err=%ld\n", - child, ceph_vinop(inode), (IS_ERR(dn) ? PTR_ERR(dn) : 0)); + child, ceph_vinop(inode), (long)PTR_ERR_OR_ZERO(dn)); return dn; } diff --git a/fs/ceph/file.c b/fs/ceph/file.c index c5517ffeb11c..685a03cc4b77 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -10,6 +10,7 @@ #include <linux/namei.h> #include <linux/writeback.h> #include <linux/falloc.h> +#include <linux/iversion.h> #include "super.h" #include "mds_client.h" @@ -437,7 +438,7 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_mds_request *req; struct dentry *dn; - struct ceph_acls_info acls = {}; + struct ceph_acl_sec_ctx as_ctx = {}; int mask; int err; @@ -451,25 +452,28 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, if (flags & O_CREAT) { if (ceph_quota_is_max_files_exceeded(dir)) return -EDQUOT; - err = ceph_pre_init_acls(dir, &mode, &acls); + err = ceph_pre_init_acls(dir, &mode, &as_ctx); if (err < 0) return err; + err = ceph_security_init_secctx(dentry, mode, &as_ctx); + if (err < 0) + goto out_ctx; } /* do the open */ req = prepare_open_request(dir->i_sb, flags, mode); if (IS_ERR(req)) { err = PTR_ERR(req); - goto out_acl; + goto out_ctx; } req->r_dentry = dget(dentry); req->r_num_caps = 2; if (flags & O_CREAT) { req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL; req->r_dentry_unless = CEPH_CAP_FILE_EXCL; - if (acls.pagelist) { - req->r_pagelist = acls.pagelist; - acls.pagelist = NULL; + if (as_ctx.pagelist) { + req->r_pagelist = as_ctx.pagelist; + as_ctx.pagelist = NULL; } } @@ -507,7 +511,7 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, } else { dout("atomic_open finish_open on dn %p\n", dn); if (req->r_op == CEPH_MDS_OP_CREATE && req->r_reply_info.has_create_ino) { - ceph_init_inode_acls(d_inode(dentry), &acls); + ceph_init_inode_acls(d_inode(dentry), &as_ctx); file->f_mode |= FMODE_CREATED; } err = finish_open(file, dentry, ceph_open); @@ -516,8 +520,8 @@ out_req: if (!req->r_err && req->r_target_inode) ceph_put_fmode(ceph_inode(req->r_target_inode), req->r_fmode); ceph_mdsc_put_request(req); -out_acl: - ceph_release_acls_info(&acls); +out_ctx: + ceph_release_acl_sec_ctx(&as_ctx); dout("atomic_open result=%d\n", err); return err; } @@ -1007,7 +1011,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter, * may block. */ truncate_inode_pages_range(inode->i_mapping, pos, - (pos+len) | (PAGE_SIZE - 1)); + PAGE_ALIGN(pos + len) - 1); req->r_mtime = mtime; } @@ -1022,7 +1026,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter, req->r_callback = ceph_aio_complete_req; req->r_inode = inode; req->r_priv = aio_req; - list_add_tail(&req->r_unsafe_item, &aio_req->osd_reqs); + list_add_tail(&req->r_private_item, &aio_req->osd_reqs); pos += len; continue; @@ -1082,8 +1086,8 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter, while (!list_empty(&osd_reqs)) { req = list_first_entry(&osd_reqs, struct ceph_osd_request, - r_unsafe_item); - list_del_init(&req->r_unsafe_item); + r_private_item); + list_del_init(&req->r_private_item); if (ret >= 0) ret = ceph_osdc_start_request(req->r_osdc, req, false); @@ -1432,6 +1436,8 @@ retry_snap: if (err) goto out; + inode_inc_iversion_raw(inode); + if (ci->i_inline_version != CEPH_INLINE_NONE) { err = ceph_uninline_data(file, NULL); if (err < 0) @@ -2063,6 +2069,8 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, do_final_copy = true; file_update_time(dst_file); + inode_inc_iversion_raw(dst_inode); + if (endoff > size) { int caps_flags = 0; diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 761451f36e2d..791f84a13bb8 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -13,6 +13,7 @@ #include <linux/posix_acl.h> #include <linux/random.h> #include <linux/sort.h> +#include <linux/iversion.h> #include "super.h" #include "mds_client.h" @@ -42,6 +43,7 @@ static int ceph_set_ino_cb(struct inode *inode, void *data) { ceph_inode(inode)->i_vino = *(struct ceph_vino *)data; inode->i_ino = ceph_vino_to_ino(*(struct ceph_vino *)data); + inode_set_iversion_raw(inode, 0); return 0; } @@ -509,6 +511,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) INIT_WORK(&ci->i_work, ceph_inode_work); ci->i_work_mask = 0; + memset(&ci->i_btime, '\0', sizeof(ci->i_btime)); ceph_fscache_inode_init(ci); @@ -523,17 +526,20 @@ void ceph_free_inode(struct inode *inode) kmem_cache_free(ceph_inode_cachep, ci); } -void ceph_destroy_inode(struct inode *inode) +void ceph_evict_inode(struct inode *inode) { struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_inode_frag *frag; struct rb_node *n; - dout("destroy_inode %p ino %llx.%llx\n", inode, ceph_vinop(inode)); + dout("evict_inode %p ino %llx.%llx\n", inode, ceph_vinop(inode)); + + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); ceph_fscache_unregister_inode_cookie(ci); - __ceph_remove_caps(inode); + __ceph_remove_caps(ci); if (__ceph_has_any_quota(ci)) ceph_adjust_quota_realms_count(inode, false); @@ -578,16 +584,6 @@ void ceph_destroy_inode(struct inode *inode) ceph_put_string(rcu_dereference_raw(ci->i_layout.pool_ns)); } -int ceph_drop_inode(struct inode *inode) -{ - /* - * Positve dentry and corresponding inode are always accompanied - * in MDS reply. So no need to keep inode in the cache after - * dropping all its aliases. - */ - return 1; -} - static inline blkcnt_t calc_inode_blocks(u64 size) { return (size + (1<<9) - 1) >> 9; @@ -795,6 +791,9 @@ static int fill_inode(struct inode *inode, struct page *locked_page, le64_to_cpu(info->version) > (ci->i_version & ~1))) new_version = true; + /* Update change_attribute */ + inode_set_max_iversion_raw(inode, iinfo->change_attr); + __ceph_caps_issued(ci, &issued); issued |= __ceph_caps_dirty(ci); new_issued = ~issued & info_caps; @@ -813,6 +812,8 @@ static int fill_inode(struct inode *inode, struct page *locked_page, dout("%p mode 0%o uid.gid %d.%d\n", inode, inode->i_mode, from_kuid(&init_user_ns, inode->i_uid), from_kgid(&init_user_ns, inode->i_gid)); + ceph_decode_timespec64(&ci->i_btime, &iinfo->btime); + ceph_decode_timespec64(&ci->i_snap_btime, &iinfo->snap_btime); } if ((new_version || (new_issued & CEPH_CAP_LINK_SHARED)) && @@ -887,6 +888,7 @@ static int fill_inode(struct inode *inode, struct page *locked_page, iinfo->xattr_data, iinfo->xattr_len); ci->i_xattrs.version = le64_to_cpu(info->xattr_version); ceph_forget_all_cached_acls(inode); + ceph_security_invalidate_secctx(inode); xattr_blob = NULL; } @@ -1027,59 +1029,38 @@ out: } /* - * caller should hold session s_mutex. + * caller should hold session s_mutex and dentry->d_lock. */ -static void update_dentry_lease(struct dentry *dentry, - struct ceph_mds_reply_lease *lease, - struct ceph_mds_session *session, - unsigned long from_time, - struct ceph_vino *tgt_vino, - struct ceph_vino *dir_vino) +static void __update_dentry_lease(struct inode *dir, struct dentry *dentry, + struct ceph_mds_reply_lease *lease, + struct ceph_mds_session *session, + unsigned long from_time, + struct ceph_mds_session **old_lease_session) { struct ceph_dentry_info *di = ceph_dentry(dentry); long unsigned duration = le32_to_cpu(lease->duration_ms); long unsigned ttl = from_time + (duration * HZ) / 1000; long unsigned half_ttl = from_time + (duration * HZ / 2) / 1000; - struct inode *dir; - struct ceph_mds_session *old_lease_session = NULL; - /* - * Make sure dentry's inode matches tgt_vino. NULL tgt_vino means that - * we expect a negative dentry. - */ - if (!tgt_vino && d_really_is_positive(dentry)) - return; - - if (tgt_vino && (d_really_is_negative(dentry) || - !ceph_ino_compare(d_inode(dentry), tgt_vino))) - return; - - spin_lock(&dentry->d_lock); dout("update_dentry_lease %p duration %lu ms ttl %lu\n", dentry, duration, ttl); - dir = d_inode(dentry->d_parent); - - /* make sure parent matches dir_vino */ - if (!ceph_ino_compare(dir, dir_vino)) - goto out_unlock; - /* only track leases on regular dentries */ if (ceph_snap(dir) != CEPH_NOSNAP) - goto out_unlock; + return; di->lease_shared_gen = atomic_read(&ceph_inode(dir)->i_shared_gen); if (duration == 0) { __ceph_dentry_dir_lease_touch(di); - goto out_unlock; + return; } if (di->lease_gen == session->s_cap_gen && time_before(ttl, di->time)) - goto out_unlock; /* we already have a newer lease. */ + return; /* we already have a newer lease. */ if (di->lease_session && di->lease_session != session) { - old_lease_session = di->lease_session; + *old_lease_session = di->lease_session; di->lease_session = NULL; } @@ -1092,6 +1073,62 @@ static void update_dentry_lease(struct dentry *dentry, di->time = ttl; __ceph_dentry_lease_touch(di); +} + +static inline void update_dentry_lease(struct inode *dir, struct dentry *dentry, + struct ceph_mds_reply_lease *lease, + struct ceph_mds_session *session, + unsigned long from_time) +{ + struct ceph_mds_session *old_lease_session = NULL; + spin_lock(&dentry->d_lock); + __update_dentry_lease(dir, dentry, lease, session, from_time, + &old_lease_session); + spin_unlock(&dentry->d_lock); + if (old_lease_session) + ceph_put_mds_session(old_lease_session); +} + +/* + * update dentry lease without having parent inode locked + */ +static void update_dentry_lease_careful(struct dentry *dentry, + struct ceph_mds_reply_lease *lease, + struct ceph_mds_session *session, + unsigned long from_time, + char *dname, u32 dname_len, + struct ceph_vino *pdvino, + struct ceph_vino *ptvino) + +{ + struct inode *dir; + struct ceph_mds_session *old_lease_session = NULL; + + spin_lock(&dentry->d_lock); + /* make sure dentry's name matches target */ + if (dentry->d_name.len != dname_len || + memcmp(dentry->d_name.name, dname, dname_len)) + goto out_unlock; + + dir = d_inode(dentry->d_parent); + /* make sure parent matches dvino */ + if (!ceph_ino_compare(dir, pdvino)) + goto out_unlock; + + /* make sure dentry's inode matches target. NULL ptvino means that + * we expect a negative dentry */ + if (ptvino) { + if (d_really_is_negative(dentry)) + goto out_unlock; + if (!ceph_ino_compare(d_inode(dentry), ptvino)) + goto out_unlock; + } else { + if (d_really_is_positive(dentry)) + goto out_unlock; + } + + __update_dentry_lease(dir, dentry, lease, session, + from_time, &old_lease_session); out_unlock: spin_unlock(&dentry->d_lock); if (old_lease_session) @@ -1156,19 +1193,6 @@ static int splice_dentry(struct dentry **pdn, struct inode *in) return 0; } -static int d_name_cmp(struct dentry *dentry, const char *name, size_t len) -{ - int ret; - - /* take d_lock to ensure dentry->d_name stability */ - spin_lock(&dentry->d_lock); - ret = dentry->d_name.len - len; - if (!ret) - ret = memcmp(dentry->d_name.name, name, len); - spin_unlock(&dentry->d_lock); - return ret; -} - /* * Incorporate results into the local cache. This is either just * one inode, or a directory, dentry, and possibly linked-to inode (e.g., @@ -1371,10 +1395,9 @@ retry_lookup: } else if (have_lease) { if (d_unhashed(dn)) d_add(dn, NULL); - update_dentry_lease(dn, rinfo->dlease, - session, - req->r_request_started, - NULL, &dvino); + update_dentry_lease(dir, dn, + rinfo->dlease, session, + req->r_request_started); } goto done; } @@ -1396,11 +1419,9 @@ retry_lookup: } if (have_lease) { - tvino.ino = le64_to_cpu(rinfo->targeti.in->ino); - tvino.snap = le64_to_cpu(rinfo->targeti.in->snapid); - update_dentry_lease(dn, rinfo->dlease, session, - req->r_request_started, - &tvino, &dvino); + update_dentry_lease(dir, dn, + rinfo->dlease, session, + req->r_request_started); } dout(" final dn %p\n", dn); } else if ((req->r_op == CEPH_MDS_OP_LOOKUPSNAP || @@ -1418,27 +1439,20 @@ retry_lookup: err = splice_dentry(&req->r_dentry, in); if (err < 0) goto done; - } else if (rinfo->head->is_dentry && - !d_name_cmp(req->r_dentry, rinfo->dname, rinfo->dname_len)) { + } else if (rinfo->head->is_dentry && req->r_dentry) { + /* parent inode is not locked, be carefull */ struct ceph_vino *ptvino = NULL; - - if ((le32_to_cpu(rinfo->diri.in->cap.caps) & CEPH_CAP_FILE_SHARED) || - le32_to_cpu(rinfo->dlease->duration_ms)) { - dvino.ino = le64_to_cpu(rinfo->diri.in->ino); - dvino.snap = le64_to_cpu(rinfo->diri.in->snapid); - - if (rinfo->head->is_target) { - tvino.ino = le64_to_cpu(rinfo->targeti.in->ino); - tvino.snap = le64_to_cpu(rinfo->targeti.in->snapid); - ptvino = &tvino; - } - - update_dentry_lease(req->r_dentry, rinfo->dlease, - session, req->r_request_started, ptvino, - &dvino); - } else { - dout("%s: no dentry lease or dir cap\n", __func__); + dvino.ino = le64_to_cpu(rinfo->diri.in->ino); + dvino.snap = le64_to_cpu(rinfo->diri.in->snapid); + if (rinfo->head->is_target) { + tvino.ino = le64_to_cpu(rinfo->targeti.in->ino); + tvino.snap = le64_to_cpu(rinfo->targeti.in->snapid); + ptvino = &tvino; } + update_dentry_lease_careful(req->r_dentry, rinfo->dlease, + session, req->r_request_started, + rinfo->dname, rinfo->dname_len, + &dvino, ptvino); } done: dout("fill_trace done err=%d\n", err); @@ -1600,7 +1614,7 @@ int ceph_readdir_prepopulate(struct ceph_mds_request *req, /* FIXME: release caps/leases if error occurs */ for (i = 0; i < rinfo->dir_nr; i++) { struct ceph_mds_reply_dir_entry *rde = rinfo->dir_entries + i; - struct ceph_vino tvino, dvino; + struct ceph_vino tvino; dname.name = rde->name; dname.len = rde->name_len; @@ -1701,9 +1715,9 @@ retry_lookup: ceph_dentry(dn)->offset = rde->offset; - dvino = ceph_vino(d_inode(parent)); - update_dentry_lease(dn, rde->lease, req->r_session, - req->r_request_started, &tvino, &dvino); + update_dentry_lease(d_inode(parent), dn, + rde->lease, req->r_session, + req->r_request_started); if (err == 0 && skipped == 0 && cache_ctl.index >= 0) { ret = fill_readdir_cache(d_inode(parent), dn, @@ -2282,7 +2296,7 @@ static int statx_to_caps(u32 want) { int mask = 0; - if (want & (STATX_MODE|STATX_UID|STATX_GID|STATX_CTIME)) + if (want & (STATX_MODE|STATX_UID|STATX_GID|STATX_CTIME|STATX_BTIME)) mask |= CEPH_CAP_AUTH_SHARED; if (want & (STATX_NLINK|STATX_CTIME)) @@ -2307,6 +2321,7 @@ int ceph_getattr(const struct path *path, struct kstat *stat, { struct inode *inode = d_inode(path->dentry); struct ceph_inode_info *ci = ceph_inode(inode); + u32 valid_mask = STATX_BASIC_STATS; int err = 0; /* Skip the getattr altogether if we're asked not to sync */ @@ -2319,6 +2334,16 @@ int ceph_getattr(const struct path *path, struct kstat *stat, generic_fillattr(inode, stat); stat->ino = ceph_translate_ino(inode->i_sb, inode->i_ino); + + /* + * btime on newly-allocated inodes is 0, so if this is still set to + * that, then assume that it's not valid. + */ + if (ci->i_btime.tv_sec || ci->i_btime.tv_nsec) { + stat->btime = ci->i_btime; + valid_mask |= STATX_BTIME; + } + if (ceph_snap(inode) == CEPH_NOSNAP) stat->dev = inode->i_sb->s_dev; else @@ -2342,7 +2367,6 @@ int ceph_getattr(const struct path *path, struct kstat *stat, stat->nlink = 1 + 1 + ci->i_subdirs; } - /* Mask off any higher bits (e.g. btime) until we have support */ - stat->result_mask = request_mask & STATX_BASIC_STATS; + stat->result_mask = request_mask & valid_mask; return err; } diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index c8a9b89b922d..920e9f048bd8 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -150,14 +150,13 @@ static int parse_reply_info_in(void **p, void *end, info->pool_ns_data = *p; *p += info->pool_ns_len; } - /* btime, change_attr */ - { - struct ceph_timespec btime; - u64 change_attr; - ceph_decode_need(p, end, sizeof(btime), bad); - ceph_decode_copy(p, &btime, sizeof(btime)); - ceph_decode_64_safe(p, end, change_attr, bad); - } + + /* btime */ + ceph_decode_need(p, end, sizeof(info->btime), bad); + ceph_decode_copy(p, &info->btime, sizeof(info->btime)); + + /* change attribute */ + ceph_decode_64_safe(p, end, info->change_attr, bad); /* dir pin */ if (struct_v >= 2) { @@ -166,6 +165,15 @@ static int parse_reply_info_in(void **p, void *end, info->dir_pin = -ENODATA; } + /* snapshot birth time, remains zero for v<=2 */ + if (struct_v >= 3) { + ceph_decode_need(p, end, sizeof(info->snap_btime), bad); + ceph_decode_copy(p, &info->snap_btime, + sizeof(info->snap_btime)); + } else { + memset(&info->snap_btime, 0, sizeof(info->snap_btime)); + } + *p = end; } else { if (features & CEPH_FEATURE_MDS_INLINE_DATA) { @@ -197,7 +205,14 @@ static int parse_reply_info_in(void **p, void *end, } } + if (features & CEPH_FEATURE_FS_BTIME) { + ceph_decode_need(p, end, sizeof(info->btime), bad); + ceph_decode_copy(p, &info->btime, sizeof(info->btime)); + ceph_decode_64_safe(p, end, info->change_attr, bad); + } + info->dir_pin = -ENODATA; + /* info->snap_btime remains zero */ } return 0; bad: @@ -717,6 +732,7 @@ void ceph_mdsc_release_request(struct kref *kref) ceph_pagelist_release(req->r_pagelist); put_request_session(req); ceph_unreserve_caps(req->r_mdsc, &req->r_caps_reservation); + WARN_ON_ONCE(!list_empty(&req->r_wait)); kfree(req); } @@ -903,7 +919,7 @@ static int __choose_mds(struct ceph_mds_client *mdsc, struct inode *dir; rcu_read_lock(); - parent = req->r_dentry->d_parent; + parent = READ_ONCE(req->r_dentry->d_parent); dir = req->r_parent ? : d_inode_rcu(parent); if (!dir || dir->i_sb != mdsc->fsc->sb) { @@ -2135,7 +2151,7 @@ retry: memcpy(path + pos, temp->d_name.name, temp->d_name.len); } spin_unlock(&temp->d_lock); - temp = temp->d_parent; + temp = READ_ONCE(temp->d_parent); /* Are we at the root? */ if (IS_ROOT(temp)) @@ -3727,42 +3743,35 @@ static void check_new_map(struct ceph_mds_client *mdsc, ceph_mdsmap_is_laggy(newmap, i) ? " (laggy)" : "", ceph_session_state_name(s->s_state)); - if (i >= newmap->m_num_mds || - memcmp(ceph_mdsmap_get_addr(oldmap, i), - ceph_mdsmap_get_addr(newmap, i), - sizeof(struct ceph_entity_addr))) { - if (s->s_state == CEPH_MDS_SESSION_OPENING) { - /* the session never opened, just close it - * out now */ - get_session(s); - __unregister_session(mdsc, s); - __wake_requests(mdsc, &s->s_waiting); - ceph_put_mds_session(s); - } else if (i >= newmap->m_num_mds) { - /* force close session for stopped mds */ - get_session(s); - __unregister_session(mdsc, s); - __wake_requests(mdsc, &s->s_waiting); - kick_requests(mdsc, i); - mutex_unlock(&mdsc->mutex); + if (i >= newmap->m_num_mds) { + /* force close session for stopped mds */ + get_session(s); + __unregister_session(mdsc, s); + __wake_requests(mdsc, &s->s_waiting); + mutex_unlock(&mdsc->mutex); - mutex_lock(&s->s_mutex); - cleanup_session_requests(mdsc, s); - remove_session_caps(s); - mutex_unlock(&s->s_mutex); + mutex_lock(&s->s_mutex); + cleanup_session_requests(mdsc, s); + remove_session_caps(s); + mutex_unlock(&s->s_mutex); - ceph_put_mds_session(s); + ceph_put_mds_session(s); - mutex_lock(&mdsc->mutex); - } else { - /* just close it */ - mutex_unlock(&mdsc->mutex); - mutex_lock(&s->s_mutex); - mutex_lock(&mdsc->mutex); - ceph_con_close(&s->s_con); - mutex_unlock(&s->s_mutex); - s->s_state = CEPH_MDS_SESSION_RESTARTING; - } + mutex_lock(&mdsc->mutex); + kick_requests(mdsc, i); + continue; + } + + if (memcmp(ceph_mdsmap_get_addr(oldmap, i), + ceph_mdsmap_get_addr(newmap, i), + sizeof(struct ceph_entity_addr))) { + /* just close it */ + mutex_unlock(&mdsc->mutex); + mutex_lock(&s->s_mutex); + mutex_lock(&mdsc->mutex); + ceph_con_close(&s->s_con); + mutex_unlock(&s->s_mutex); + s->s_state = CEPH_MDS_SESSION_RESTARTING; } else if (oldstate == newstate) { continue; /* nothing new with this mds */ } @@ -3931,31 +3940,33 @@ bad: } void ceph_mdsc_lease_send_msg(struct ceph_mds_session *session, - struct inode *inode, struct dentry *dentry, char action, u32 seq) { struct ceph_msg *msg; struct ceph_mds_lease *lease; - int len = sizeof(*lease) + sizeof(u32); - int dnamelen = 0; + struct inode *dir; + int len = sizeof(*lease) + sizeof(u32) + NAME_MAX; - dout("lease_send_msg inode %p dentry %p %s to mds%d\n", - inode, dentry, ceph_lease_op_name(action), session->s_mds); - dnamelen = dentry->d_name.len; - len += dnamelen; + dout("lease_send_msg identry %p %s to mds%d\n", + dentry, ceph_lease_op_name(action), session->s_mds); msg = ceph_msg_new(CEPH_MSG_CLIENT_LEASE, len, GFP_NOFS, false); if (!msg) return; lease = msg->front.iov_base; lease->action = action; - lease->ino = cpu_to_le64(ceph_vino(inode).ino); - lease->first = lease->last = cpu_to_le64(ceph_vino(inode).snap); lease->seq = cpu_to_le32(seq); - put_unaligned_le32(dnamelen, lease + 1); - memcpy((void *)(lease + 1) + 4, dentry->d_name.name, dnamelen); + spin_lock(&dentry->d_lock); + dir = d_inode(dentry->d_parent); + lease->ino = cpu_to_le64(ceph_ino(dir)); + lease->first = lease->last = cpu_to_le64(ceph_snap(dir)); + + put_unaligned_le32(dentry->d_name.len, lease + 1); + memcpy((void *)(lease + 1) + 4, + dentry->d_name.name, dentry->d_name.len); + spin_unlock(&dentry->d_lock); /* * if this is a preemptive lease RELEASE, no need to * flush request stream, since the actual request will @@ -4157,6 +4168,7 @@ static void wait_requests(struct ceph_mds_client *mdsc) while ((req = __get_oldest_req(mdsc))) { dout("wait_requests timed out on tid %llu\n", req->r_tid); + list_del_init(&req->r_wait); __unregister_request(mdsc, req); } } diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index a83f28bc2387..f7c8603484fe 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -69,6 +69,9 @@ struct ceph_mds_reply_info_in { u64 max_bytes; u64 max_files; s32 dir_pin; + struct ceph_timespec btime; + struct ceph_timespec snap_btime; + u64 change_attr; }; struct ceph_mds_reply_dir_entry { @@ -504,7 +507,6 @@ extern char *ceph_mdsc_build_path(struct dentry *dentry, int *plen, u64 *base, extern void __ceph_mdsc_drop_dentry_lease(struct dentry *dentry); extern void ceph_mdsc_lease_send_msg(struct ceph_mds_session *session, - struct inode *inode, struct dentry *dentry, char action, u32 seq); diff --git a/fs/ceph/mdsmap.c b/fs/ceph/mdsmap.c index 701b4fb0fb5a..ce2d00da5096 100644 --- a/fs/ceph/mdsmap.c +++ b/fs/ceph/mdsmap.c @@ -107,7 +107,7 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end) struct ceph_mdsmap *m; const void *start = *p; int i, j, n; - int err = -EINVAL; + int err; u8 mdsmap_v, mdsmap_cv; u16 mdsmap_ev; @@ -183,8 +183,9 @@ struct ceph_mdsmap *ceph_mdsmap_decode(void **p, void *end) inc = ceph_decode_32(p); state = ceph_decode_32(p); state_seq = ceph_decode_64(p); - ceph_decode_copy(p, &addr, sizeof(addr)); - ceph_decode_addr(&addr); + err = ceph_decode_entity_addr(p, end, &addr); + if (err) + goto corrupt; ceph_decode_copy(p, &laggy_since, sizeof(laggy_since)); *p += sizeof(u32); ceph_decode_32_safe(p, end, namelen, bad); @@ -357,7 +358,7 @@ bad_ext: nomem: err = -ENOMEM; goto out_err; -bad: +corrupt: pr_err("corrupt mdsmap\n"); print_hex_dump(KERN_DEBUG, "mdsmap: ", DUMP_PREFIX_OFFSET, 16, 1, @@ -365,6 +366,9 @@ bad: out_err: ceph_mdsmap_destroy(m); return ERR_PTR(err); +bad: + err = -EINVAL; + goto corrupt; } void ceph_mdsmap_destroy(struct ceph_mdsmap *m) diff --git a/fs/ceph/quota.c b/fs/ceph/quota.c index d629fc857450..de56dee60540 100644 --- a/fs/ceph/quota.c +++ b/fs/ceph/quota.c @@ -135,7 +135,7 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc, return NULL; mutex_lock(&qri->mutex); - if (qri->inode) { + if (qri->inode && ceph_is_any_caps(qri->inode)) { /* A request has already returned the inode */ mutex_unlock(&qri->mutex); return qri->inode; @@ -146,7 +146,18 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc, mutex_unlock(&qri->mutex); return NULL; } - in = ceph_lookup_inode(sb, realm->ino); + if (qri->inode) { + /* get caps */ + int ret = __ceph_do_getattr(qri->inode, NULL, + CEPH_STAT_CAP_INODE, true); + if (ret >= 0) + in = qri->inode; + else + in = ERR_PTR(ret); + } else { + in = ceph_lookup_inode(sb, realm->ino); + } + if (IS_ERR(in)) { pr_warn("Can't lookup inode %llx (err: %ld)\n", realm->ino, PTR_ERR(in)); diff --git a/fs/ceph/snap.c b/fs/ceph/snap.c index 72c6c022f02b..4c6494eb02b5 100644 --- a/fs/ceph/snap.c +++ b/fs/ceph/snap.c @@ -3,6 +3,7 @@ #include <linux/sort.h> #include <linux/slab.h> +#include <linux/iversion.h> #include "super.h" #include "mds_client.h" #include <linux/ceph/decode.h> @@ -606,6 +607,8 @@ int __ceph_finish_cap_snap(struct ceph_inode_info *ci, capsnap->mtime = inode->i_mtime; capsnap->atime = inode->i_atime; capsnap->ctime = inode->i_ctime; + capsnap->btime = ci->i_btime; + capsnap->change_attr = inode_peek_iversion_raw(inode); capsnap->time_warp_seq = ci->i_time_warp_seq; capsnap->truncate_size = ci->i_truncate_size; capsnap->truncate_seq = ci->i_truncate_seq; diff --git a/fs/ceph/super.c b/fs/ceph/super.c index ed1b65a6c2c3..ab4868c7308e 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -840,10 +840,10 @@ static int ceph_remount(struct super_block *sb, int *flags, char *data) static const struct super_operations ceph_super_ops = { .alloc_inode = ceph_alloc_inode, - .destroy_inode = ceph_destroy_inode, .free_inode = ceph_free_inode, .write_inode = ceph_write_inode, - .drop_inode = ceph_drop_inode, + .drop_inode = generic_delete_inode, + .evict_inode = ceph_evict_inode, .sync_fs = ceph_sync_fs, .put_super = ceph_put_super, .remount_fs = ceph_remount, @@ -978,7 +978,7 @@ static int ceph_set_super(struct super_block *s, void *data) s->s_d_op = &ceph_dentry_ops; s->s_export_op = &ceph_export_ops; - s->s_time_gran = 1000; /* 1000 ns == 1 us */ + s->s_time_gran = 1; ret = set_anon_super(s, NULL); /* what is that second arg for? */ if (ret != 0) @@ -1159,17 +1159,15 @@ static int __init init_ceph(void) goto out; ceph_flock_init(); - ceph_xattr_init(); ret = register_filesystem(&ceph_fs_type); if (ret) - goto out_xattr; + goto out_caches; pr_info("loaded (mds proto %d)\n", CEPH_MDSC_PROTOCOL); return 0; -out_xattr: - ceph_xattr_exit(); +out_caches: destroy_caches(); out: return ret; @@ -1179,7 +1177,6 @@ static void __exit exit_ceph(void) { dout("exit_ceph\n"); unregister_filesystem(&ceph_fs_type); - ceph_xattr_exit(); destroy_caches(); } diff --git a/fs/ceph/super.h b/fs/ceph/super.h index fbe6869a3f95..d2352fd95dbc 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -197,7 +197,8 @@ struct ceph_cap_snap { u64 xattr_version; u64 size; - struct timespec64 mtime, atime, ctime; + u64 change_attr; + struct timespec64 mtime, atime, ctime, btime; u64 time_warp_seq; u64 truncate_size; u32 truncate_seq; @@ -384,6 +385,8 @@ struct ceph_inode_info { int i_snap_realm_counter; /* snap realm (if caps) */ struct list_head i_snap_realm_item; struct list_head i_snap_flush_item; + struct timespec64 i_btime; + struct timespec64 i_snap_btime; struct work_struct i_work; unsigned long i_work_mask; @@ -544,7 +547,12 @@ static inline void __ceph_dir_set_complete(struct ceph_inode_info *ci, long long release_count, long long ordered_count) { - smp_mb__before_atomic(); + /* + * Makes sure operations that setup readdir cache (update page + * cache and i_size) are strongly ordered w.r.t. the following + * atomic64_set() operations. + */ + smp_mb(); atomic64_set(&ci->i_complete_seq[0], release_count); atomic64_set(&ci->i_complete_seq[1], ordered_count); } @@ -876,9 +884,8 @@ static inline bool __ceph_have_pending_cap_snap(struct ceph_inode_info *ci) extern const struct inode_operations ceph_file_iops; extern struct inode *ceph_alloc_inode(struct super_block *sb); -extern void ceph_destroy_inode(struct inode *inode); +extern void ceph_evict_inode(struct inode *inode); extern void ceph_free_inode(struct inode *inode); -extern int ceph_drop_inode(struct inode *inode); extern struct inode *ceph_get_inode(struct super_block *sb, struct ceph_vino vino); @@ -921,10 +928,20 @@ ssize_t __ceph_getxattr(struct inode *, const char *, void *, size_t); extern ssize_t ceph_listxattr(struct dentry *, char *, size_t); extern void __ceph_build_xattrs_blob(struct ceph_inode_info *ci); extern void __ceph_destroy_xattrs(struct ceph_inode_info *ci); -extern void __init ceph_xattr_init(void); -extern void ceph_xattr_exit(void); extern const struct xattr_handler *ceph_xattr_handlers[]; +struct ceph_acl_sec_ctx { +#ifdef CONFIG_CEPH_FS_POSIX_ACL + void *default_acl; + void *acl; +#endif +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL + void *sec_ctx; + u32 sec_ctxlen; +#endif + struct ceph_pagelist *pagelist; +}; + #ifdef CONFIG_SECURITY extern bool ceph_security_xattr_deadlock(struct inode *in); extern bool ceph_security_xattr_wanted(struct inode *in); @@ -939,21 +956,32 @@ static inline bool ceph_security_xattr_wanted(struct inode *in) } #endif -/* acl.c */ -struct ceph_acls_info { - void *default_acl; - void *acl; - struct ceph_pagelist *pagelist; -}; +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL +extern int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, + struct ceph_acl_sec_ctx *ctx); +extern void ceph_security_invalidate_secctx(struct inode *inode); +#else +static inline int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, + struct ceph_acl_sec_ctx *ctx) +{ + return 0; +} +static inline void ceph_security_invalidate_secctx(struct inode *inode) +{ +} +#endif + +void ceph_release_acl_sec_ctx(struct ceph_acl_sec_ctx *as_ctx); +/* acl.c */ #ifdef CONFIG_CEPH_FS_POSIX_ACL struct posix_acl *ceph_get_acl(struct inode *, int); int ceph_set_acl(struct inode *inode, struct posix_acl *acl, int type); int ceph_pre_init_acls(struct inode *dir, umode_t *mode, - struct ceph_acls_info *info); -void ceph_init_inode_acls(struct inode *inode, struct ceph_acls_info *info); -void ceph_release_acls_info(struct ceph_acls_info *info); + struct ceph_acl_sec_ctx *as_ctx); +void ceph_init_inode_acls(struct inode *inode, + struct ceph_acl_sec_ctx *as_ctx); static inline void ceph_forget_all_cached_acls(struct inode *inode) { @@ -966,15 +994,12 @@ static inline void ceph_forget_all_cached_acls(struct inode *inode) #define ceph_set_acl NULL static inline int ceph_pre_init_acls(struct inode *dir, umode_t *mode, - struct ceph_acls_info *info) + struct ceph_acl_sec_ctx *as_ctx) { return 0; } static inline void ceph_init_inode_acls(struct inode *inode, - struct ceph_acls_info *info) -{ -} -static inline void ceph_release_acls_info(struct ceph_acls_info *info) + struct ceph_acl_sec_ctx *as_ctx) { } static inline int ceph_acl_chmod(struct dentry *dentry, struct inode *inode) @@ -1000,7 +1025,7 @@ extern void ceph_add_cap(struct inode *inode, unsigned cap, unsigned seq, u64 realmino, int flags, struct ceph_cap **new_cap); extern void __ceph_remove_cap(struct ceph_cap *cap, bool queue_release); -extern void __ceph_remove_caps(struct inode* inode); +extern void __ceph_remove_caps(struct ceph_inode_info *ci); extern void ceph_put_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap); extern int ceph_is_any_caps(struct inode *inode); diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index 0cc42c8879e9..37b458a9af3a 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -8,6 +8,7 @@ #include <linux/ceph/decode.h> #include <linux/xattr.h> +#include <linux/security.h> #include <linux/posix_acl_xattr.h> #include <linux/slab.h> @@ -17,26 +18,9 @@ static int __remove_xattr(struct ceph_inode_info *ci, struct ceph_inode_xattr *xattr); -static const struct xattr_handler ceph_other_xattr_handler; - -/* - * List of handlers for synthetic system.* attributes. Other - * attributes are handled directly. - */ -const struct xattr_handler *ceph_xattr_handlers[] = { -#ifdef CONFIG_CEPH_FS_POSIX_ACL - &posix_acl_access_xattr_handler, - &posix_acl_default_xattr_handler, -#endif - &ceph_other_xattr_handler, - NULL, -}; - static bool ceph_is_valid_xattr(const char *name) { return !strncmp(name, XATTR_CEPH_PREFIX, XATTR_CEPH_PREFIX_LEN) || - !strncmp(name, XATTR_SECURITY_PREFIX, - XATTR_SECURITY_PREFIX_LEN) || !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) || !strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); } @@ -48,8 +32,8 @@ static bool ceph_is_valid_xattr(const char *name) struct ceph_vxattr { char *name; size_t name_size; /* strlen(name) + 1 (for '\0') */ - size_t (*getxattr_cb)(struct ceph_inode_info *ci, char *val, - size_t size); + ssize_t (*getxattr_cb)(struct ceph_inode_info *ci, char *val, + size_t size); bool (*exists_cb)(struct ceph_inode_info *ci); unsigned int flags; }; @@ -68,8 +52,8 @@ static bool ceph_vxattrcb_layout_exists(struct ceph_inode_info *ci) rcu_dereference_raw(fl->pool_ns) != NULL); } -static size_t ceph_vxattrcb_layout(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_layout(struct ceph_inode_info *ci, char *val, + size_t size) { struct ceph_fs_client *fsc = ceph_sb_to_client(ci->vfs_inode.i_sb); struct ceph_osd_client *osdc = &fsc->client->osdc; @@ -79,7 +63,7 @@ static size_t ceph_vxattrcb_layout(struct ceph_inode_info *ci, char *val, const char *ns_field = " pool_namespace="; char buf[128]; size_t len, total_len = 0; - int ret; + ssize_t ret; pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); @@ -96,18 +80,15 @@ static size_t ceph_vxattrcb_layout(struct ceph_inode_info *ci, char *val, len = snprintf(buf, sizeof(buf), "stripe_unit=%u stripe_count=%u object_size=%u pool=%lld", ci->i_layout.stripe_unit, ci->i_layout.stripe_count, - ci->i_layout.object_size, (unsigned long long)pool); + ci->i_layout.object_size, pool); total_len = len; } if (pool_ns) total_len += strlen(ns_field) + pool_ns->len; - if (!size) { - ret = total_len; - } else if (total_len > size) { - ret = -ERANGE; - } else { + ret = total_len; + if (size >= total_len) { memcpy(val, buf, len); ret = len; if (pool_name) { @@ -128,28 +109,55 @@ static size_t ceph_vxattrcb_layout(struct ceph_inode_info *ci, char *val, return ret; } -static size_t ceph_vxattrcb_layout_stripe_unit(struct ceph_inode_info *ci, - char *val, size_t size) +/* + * The convention with strings in xattrs is that they should not be NULL + * terminated, since we're returning the length with them. snprintf always + * NULL terminates however, so call it on a temporary buffer and then memcpy + * the result into place. + */ +static int ceph_fmt_xattr(char *val, size_t size, const char *fmt, ...) { - return snprintf(val, size, "%u", ci->i_layout.stripe_unit); + int ret; + va_list args; + char buf[96]; /* NB: reevaluate size if new vxattrs are added */ + + va_start(args, fmt); + ret = vsnprintf(buf, size ? sizeof(buf) : 0, fmt, args); + va_end(args); + + /* Sanity check */ + if (size && ret + 1 > sizeof(buf)) { + WARN_ONCE(true, "Returned length too big (%d)", ret); + return -E2BIG; + } + + if (ret <= size) + memcpy(val, buf, ret); + return ret; } -static size_t ceph_vxattrcb_layout_stripe_count(struct ceph_inode_info *ci, +static ssize_t ceph_vxattrcb_layout_stripe_unit(struct ceph_inode_info *ci, char *val, size_t size) { - return snprintf(val, size, "%u", ci->i_layout.stripe_count); + return ceph_fmt_xattr(val, size, "%u", ci->i_layout.stripe_unit); +} + +static ssize_t ceph_vxattrcb_layout_stripe_count(struct ceph_inode_info *ci, + char *val, size_t size) +{ + return ceph_fmt_xattr(val, size, "%u", ci->i_layout.stripe_count); } -static size_t ceph_vxattrcb_layout_object_size(struct ceph_inode_info *ci, - char *val, size_t size) +static ssize_t ceph_vxattrcb_layout_object_size(struct ceph_inode_info *ci, + char *val, size_t size) { - return snprintf(val, size, "%u", ci->i_layout.object_size); + return ceph_fmt_xattr(val, size, "%u", ci->i_layout.object_size); } -static size_t ceph_vxattrcb_layout_pool(struct ceph_inode_info *ci, - char *val, size_t size) +static ssize_t ceph_vxattrcb_layout_pool(struct ceph_inode_info *ci, + char *val, size_t size) { - int ret; + ssize_t ret; struct ceph_fs_client *fsc = ceph_sb_to_client(ci->vfs_inode.i_sb); struct ceph_osd_client *osdc = &fsc->client->osdc; s64 pool = ci->i_layout.pool_id; @@ -157,21 +165,27 @@ static size_t ceph_vxattrcb_layout_pool(struct ceph_inode_info *ci, down_read(&osdc->lock); pool_name = ceph_pg_pool_name_by_id(osdc->osdmap, pool); - if (pool_name) - ret = snprintf(val, size, "%s", pool_name); - else - ret = snprintf(val, size, "%lld", (unsigned long long)pool); + if (pool_name) { + ret = strlen(pool_name); + if (ret <= size) + memcpy(val, pool_name, ret); + } else { + ret = ceph_fmt_xattr(val, size, "%lld", pool); + } up_read(&osdc->lock); return ret; } -static size_t ceph_vxattrcb_layout_pool_namespace(struct ceph_inode_info *ci, - char *val, size_t size) +static ssize_t ceph_vxattrcb_layout_pool_namespace(struct ceph_inode_info *ci, + char *val, size_t size) { - int ret = 0; + ssize_t ret = 0; struct ceph_string *ns = ceph_try_get_string(ci->i_layout.pool_ns); + if (ns) { - ret = snprintf(val, size, "%.*s", (int)ns->len, ns->str); + ret = ns->len; + if (ret <= size) + memcpy(val, ns->str, ret); ceph_put_string(ns); } return ret; @@ -179,53 +193,54 @@ static size_t ceph_vxattrcb_layout_pool_namespace(struct ceph_inode_info *ci, /* directories */ -static size_t ceph_vxattrcb_dir_entries(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_entries(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld", ci->i_files + ci->i_subdirs); + return ceph_fmt_xattr(val, size, "%lld", ci->i_files + ci->i_subdirs); } -static size_t ceph_vxattrcb_dir_files(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_files(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld", ci->i_files); + return ceph_fmt_xattr(val, size, "%lld", ci->i_files); } -static size_t ceph_vxattrcb_dir_subdirs(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_subdirs(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld", ci->i_subdirs); + return ceph_fmt_xattr(val, size, "%lld", ci->i_subdirs); } -static size_t ceph_vxattrcb_dir_rentries(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_rentries(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld", ci->i_rfiles + ci->i_rsubdirs); + return ceph_fmt_xattr(val, size, "%lld", + ci->i_rfiles + ci->i_rsubdirs); } -static size_t ceph_vxattrcb_dir_rfiles(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_rfiles(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld", ci->i_rfiles); + return ceph_fmt_xattr(val, size, "%lld", ci->i_rfiles); } -static size_t ceph_vxattrcb_dir_rsubdirs(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_rsubdirs(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld", ci->i_rsubdirs); + return ceph_fmt_xattr(val, size, "%lld", ci->i_rsubdirs); } -static size_t ceph_vxattrcb_dir_rbytes(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_rbytes(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld", ci->i_rbytes); + return ceph_fmt_xattr(val, size, "%lld", ci->i_rbytes); } -static size_t ceph_vxattrcb_dir_rctime(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_rctime(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%lld.09%ld", ci->i_rctime.tv_sec, - ci->i_rctime.tv_nsec); + return ceph_fmt_xattr(val, size, "%lld.%09ld", ci->i_rctime.tv_sec, + ci->i_rctime.tv_nsec); } /* dir pin */ @@ -234,10 +249,10 @@ static bool ceph_vxattrcb_dir_pin_exists(struct ceph_inode_info *ci) return ci->i_dir_pin != -ENODATA; } -static size_t ceph_vxattrcb_dir_pin(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_dir_pin(struct ceph_inode_info *ci, char *val, + size_t size) { - return snprintf(val, size, "%d", (int)ci->i_dir_pin); + return ceph_fmt_xattr(val, size, "%d", (int)ci->i_dir_pin); } /* quotas */ @@ -254,23 +269,36 @@ static bool ceph_vxattrcb_quota_exists(struct ceph_inode_info *ci) return ret; } -static size_t ceph_vxattrcb_quota(struct ceph_inode_info *ci, char *val, - size_t size) +static ssize_t ceph_vxattrcb_quota(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "max_bytes=%llu max_files=%llu", + ci->i_max_bytes, ci->i_max_files); +} + +static ssize_t ceph_vxattrcb_quota_max_bytes(struct ceph_inode_info *ci, + char *val, size_t size) { - return snprintf(val, size, "max_bytes=%llu max_files=%llu", - ci->i_max_bytes, ci->i_max_files); + return ceph_fmt_xattr(val, size, "%llu", ci->i_max_bytes); } -static size_t ceph_vxattrcb_quota_max_bytes(struct ceph_inode_info *ci, - char *val, size_t size) +static ssize_t ceph_vxattrcb_quota_max_files(struct ceph_inode_info *ci, + char *val, size_t size) { - return snprintf(val, size, "%llu", ci->i_max_bytes); + return ceph_fmt_xattr(val, size, "%llu", ci->i_max_files); } -static size_t ceph_vxattrcb_quota_max_files(struct ceph_inode_info *ci, - char *val, size_t size) +/* snapshots */ +static bool ceph_vxattrcb_snap_btime_exists(struct ceph_inode_info *ci) { - return snprintf(val, size, "%llu", ci->i_max_files); + return (ci->i_snap_btime.tv_sec != 0 || ci->i_snap_btime.tv_nsec != 0); +} + +static ssize_t ceph_vxattrcb_snap_btime(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld.%09ld", ci->i_snap_btime.tv_sec, + ci->i_snap_btime.tv_nsec); } #define CEPH_XATTR_NAME(_type, _name) XATTR_CEPH_PREFIX #_type "." #_name @@ -327,7 +355,7 @@ static struct ceph_vxattr ceph_dir_vxattrs[] = { XATTR_RSTAT_FIELD(dir, rctime), { .name = "ceph.dir.pin", - .name_size = sizeof("ceph.dir_pin"), + .name_size = sizeof("ceph.dir.pin"), .getxattr_cb = ceph_vxattrcb_dir_pin, .exists_cb = ceph_vxattrcb_dir_pin_exists, .flags = VXATTR_FLAG_HIDDEN, @@ -341,9 +369,15 @@ static struct ceph_vxattr ceph_dir_vxattrs[] = { }, XATTR_QUOTA_FIELD(quota, max_bytes), XATTR_QUOTA_FIELD(quota, max_files), + { + .name = "ceph.snap.btime", + .name_size = sizeof("ceph.snap.btime"), + .getxattr_cb = ceph_vxattrcb_snap_btime, + .exists_cb = ceph_vxattrcb_snap_btime_exists, + .flags = VXATTR_FLAG_READONLY, + }, { .name = NULL, 0 } /* Required table terminator */ }; -static size_t ceph_dir_vxattrs_name_size; /* total size of all names */ /* files */ @@ -360,9 +394,15 @@ static struct ceph_vxattr ceph_file_vxattrs[] = { XATTR_LAYOUT_FIELD(file, layout, object_size), XATTR_LAYOUT_FIELD(file, layout, pool), XATTR_LAYOUT_FIELD(file, layout, pool_namespace), + { + .name = "ceph.snap.btime", + .name_size = sizeof("ceph.snap.btime"), + .getxattr_cb = ceph_vxattrcb_snap_btime, + .exists_cb = ceph_vxattrcb_snap_btime_exists, + .flags = VXATTR_FLAG_READONLY, + }, { .name = NULL, 0 } /* Required table terminator */ }; -static size_t ceph_file_vxattrs_name_size; /* total size of all names */ static struct ceph_vxattr *ceph_inode_vxattrs(struct inode *inode) { @@ -373,47 +413,6 @@ static struct ceph_vxattr *ceph_inode_vxattrs(struct inode *inode) return NULL; } -static size_t ceph_vxattrs_name_size(struct ceph_vxattr *vxattrs) -{ - if (vxattrs == ceph_dir_vxattrs) - return ceph_dir_vxattrs_name_size; - if (vxattrs == ceph_file_vxattrs) - return ceph_file_vxattrs_name_size; - BUG_ON(vxattrs); - return 0; -} - -/* - * Compute the aggregate size (including terminating '\0') of all - * virtual extended attribute names in the given vxattr table. - */ -static size_t __init vxattrs_name_size(struct ceph_vxattr *vxattrs) -{ - struct ceph_vxattr *vxattr; - size_t size = 0; - - for (vxattr = vxattrs; vxattr->name; vxattr++) { - if (!(vxattr->flags & VXATTR_FLAG_HIDDEN)) - size += vxattr->name_size; - } - - return size; -} - -/* Routines called at initialization and exit time */ - -void __init ceph_xattr_init(void) -{ - ceph_dir_vxattrs_name_size = vxattrs_name_size(ceph_dir_vxattrs); - ceph_file_vxattrs_name_size = vxattrs_name_size(ceph_file_vxattrs); -} - -void ceph_xattr_exit(void) -{ - ceph_dir_vxattrs_name_size = 0; - ceph_file_vxattrs_name_size = 0; -} - static struct ceph_vxattr *ceph_match_vxattr(struct inode *inode, const char *name) { @@ -523,8 +522,8 @@ static int __set_xattr(struct ceph_inode_info *ci, dout("__set_xattr_val p=%p\n", p); } - dout("__set_xattr_val added %llx.%llx xattr %p %s=%.*s\n", - ceph_vinop(&ci->vfs_inode), xattr, name, val_len, val); + dout("__set_xattr_val added %llx.%llx xattr %p %.*s=%.*s\n", + ceph_vinop(&ci->vfs_inode), xattr, name_len, name, val_len, val); return 0; } @@ -823,7 +822,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, struct ceph_inode_xattr *xattr; struct ceph_vxattr *vxattr = NULL; int req_mask; - int err; + ssize_t err; /* let's see if a virtual xattr was requested */ vxattr = ceph_match_vxattr(inode, name); @@ -835,8 +834,11 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, if (err) return err; err = -ENODATA; - if (!(vxattr->exists_cb && !vxattr->exists_cb(ci))) + if (!(vxattr->exists_cb && !vxattr->exists_cb(ci))) { err = vxattr->getxattr_cb(ci, value, size); + if (size && size < err) + err = -ERANGE; + } return err; } @@ -897,10 +899,9 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size) struct inode *inode = d_inode(dentry); struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_vxattr *vxattrs = ceph_inode_vxattrs(inode); - u32 vir_namelen = 0; + bool len_only = (size == 0); u32 namelen; int err; - u32 len; int i; spin_lock(&ci->i_ceph_lock); @@ -919,38 +920,45 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size) err = __build_xattrs(inode); if (err < 0) goto out; - /* - * Start with virtual dir xattr names (if any) (including - * terminating '\0' characters for each). - */ - vir_namelen = ceph_vxattrs_name_size(vxattrs); - /* adding 1 byte per each variable due to the null termination */ + /* add 1 byte for each xattr due to the null termination */ namelen = ci->i_xattrs.names_size + ci->i_xattrs.count; - err = -ERANGE; - if (size && vir_namelen + namelen > size) - goto out; - - err = namelen + vir_namelen; - if (size == 0) - goto out; + if (!len_only) { + if (namelen > size) { + err = -ERANGE; + goto out; + } + names = __copy_xattr_names(ci, names); + size -= namelen; + } - names = __copy_xattr_names(ci, names); /* virtual xattr names, too */ - err = namelen; if (vxattrs) { for (i = 0; vxattrs[i].name; i++) { - if (!(vxattrs[i].flags & VXATTR_FLAG_HIDDEN) && - !(vxattrs[i].exists_cb && - !vxattrs[i].exists_cb(ci))) { - len = sprintf(names, "%s", vxattrs[i].name); - names += len + 1; - err += len + 1; + size_t this_len; + + if (vxattrs[i].flags & VXATTR_FLAG_HIDDEN) + continue; + if (vxattrs[i].exists_cb && !vxattrs[i].exists_cb(ci)) + continue; + + this_len = strlen(vxattrs[i].name) + 1; + namelen += this_len; + if (len_only) + continue; + + if (this_len > size) { + err = -ERANGE; + goto out; } + + memcpy(names, vxattrs[i].name, this_len); + names += this_len; + size -= this_len; } } - + err = namelen; out: spin_unlock(&ci->i_ceph_lock); return err; @@ -1206,4 +1214,138 @@ bool ceph_security_xattr_deadlock(struct inode *in) spin_unlock(&ci->i_ceph_lock); return ret; } + +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL +int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, + struct ceph_acl_sec_ctx *as_ctx) +{ + struct ceph_pagelist *pagelist = as_ctx->pagelist; + const char *name; + size_t name_len; + int err; + + err = security_dentry_init_security(dentry, mode, &dentry->d_name, + &as_ctx->sec_ctx, + &as_ctx->sec_ctxlen); + if (err < 0) { + WARN_ON_ONCE(err != -EOPNOTSUPP); + err = 0; /* do nothing */ + goto out; + } + + err = -ENOMEM; + if (!pagelist) { + pagelist = ceph_pagelist_alloc(GFP_KERNEL); + if (!pagelist) + goto out; + err = ceph_pagelist_reserve(pagelist, PAGE_SIZE); + if (err) + goto out; + ceph_pagelist_encode_32(pagelist, 1); + } + + /* + * FIXME: Make security_dentry_init_security() generic. Currently + * It only supports single security module and only selinux has + * dentry_init_security hook. + */ + name = XATTR_NAME_SELINUX; + name_len = strlen(name); + err = ceph_pagelist_reserve(pagelist, + 4 * 2 + name_len + as_ctx->sec_ctxlen); + if (err) + goto out; + + if (as_ctx->pagelist) { + /* update count of KV pairs */ + BUG_ON(pagelist->length <= sizeof(__le32)); + if (list_is_singular(&pagelist->head)) { + le32_add_cpu((__le32*)pagelist->mapped_tail, 1); + } else { + struct page *page = list_first_entry(&pagelist->head, + struct page, lru); + void *addr = kmap_atomic(page); + le32_add_cpu((__le32*)addr, 1); + kunmap_atomic(addr); + } + } else { + as_ctx->pagelist = pagelist; + } + + ceph_pagelist_encode_32(pagelist, name_len); + ceph_pagelist_append(pagelist, name, name_len); + + ceph_pagelist_encode_32(pagelist, as_ctx->sec_ctxlen); + ceph_pagelist_append(pagelist, as_ctx->sec_ctx, as_ctx->sec_ctxlen); + + err = 0; +out: + if (pagelist && !as_ctx->pagelist) + ceph_pagelist_release(pagelist); + return err; +} + +void ceph_security_invalidate_secctx(struct inode *inode) +{ + security_inode_invalidate_secctx(inode); +} + +static int ceph_xattr_set_security_label(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *key, const void *buf, + size_t buflen, int flags) +{ + if (security_ismaclabel(key)) { + const char *name = xattr_full_name(handler, key); + return __ceph_setxattr(inode, name, buf, buflen, flags); + } + return -EOPNOTSUPP; +} + +static int ceph_xattr_get_security_label(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *key, void *buf, size_t buflen) +{ + if (security_ismaclabel(key)) { + const char *name = xattr_full_name(handler, key); + return __ceph_getxattr(inode, name, buf, buflen); + } + return -EOPNOTSUPP; +} + +static const struct xattr_handler ceph_security_label_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = ceph_xattr_get_security_label, + .set = ceph_xattr_set_security_label, +}; +#endif #endif + +void ceph_release_acl_sec_ctx(struct ceph_acl_sec_ctx *as_ctx) +{ +#ifdef CONFIG_CEPH_FS_POSIX_ACL + posix_acl_release(as_ctx->acl); + posix_acl_release(as_ctx->default_acl); +#endif +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL + security_release_secctx(as_ctx->sec_ctx, as_ctx->sec_ctxlen); +#endif + if (as_ctx->pagelist) + ceph_pagelist_release(as_ctx->pagelist); +} + +/* + * List of handlers for synthetic system.* attributes. Other + * attributes are handled directly. + */ +const struct xattr_handler *ceph_xattr_handlers[] = { +#ifdef CONFIG_CEPH_FS_POSIX_ACL + &posix_acl_access_xattr_handler, + &posix_acl_default_xattr_handler, +#endif +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL + &ceph_security_label_handler, +#endif + &ceph_other_xattr_handler, + NULL, +}; diff --git a/fs/cifs/Kconfig b/fs/cifs/Kconfig index 523e9ea78a28..b16219e5dac9 100644 --- a/fs/cifs/Kconfig +++ b/fs/cifs/Kconfig @@ -13,9 +13,11 @@ config CIFS select CRYPTO_LIB_ARC4 select CRYPTO_AEAD2 select CRYPTO_CCM + select CRYPTO_GCM select CRYPTO_ECB select CRYPTO_AES select CRYPTO_DES + select KEYS help This is the client VFS module for the SMB3 family of NAS protocols, (including support for the most recent, most secure dialect SMB3.1.1) @@ -109,7 +111,7 @@ config CIFS_WEAK_PW_HASH config CIFS_UPCALL bool "Kerberos/SPNEGO advanced session setup" - depends on CIFS && KEYS + depends on CIFS select DNS_RESOLVER help Enables an upcall mechanism for CIFS which accesses userspace helper @@ -144,14 +146,6 @@ config CIFS_POSIX (such as Samba 3.10 and later) which can negotiate CIFS POSIX ACL support. If unsure, say N. -config CIFS_ACL - bool "Provide CIFS ACL support" - depends on CIFS_XATTR && KEYS - help - Allows fetching CIFS/NTFS ACL from the server. The DACL blob - is handed over to the application/caller. See the man - page for getcifsacl for more information. If unsure, say Y. - config CIFS_DEBUG bool "Enable CIFS debugging routines" default y @@ -184,7 +178,7 @@ config CIFS_DEBUG_DUMP_KEYS config CIFS_DFS_UPCALL bool "DFS feature support" - depends on CIFS && KEYS + depends on CIFS select DNS_RESOLVER help Distributed File System (DFS) support is used to access shares @@ -203,10 +197,10 @@ config CIFS_NFSD_EXPORT Allows NFS server to export a CIFS mounted share (nfsd over cifs) config CIFS_SMB_DIRECT - bool "SMB Direct support (Experimental)" + bool "SMB Direct support" depends on CIFS=m && INFINIBAND && INFINIBAND_ADDR_TRANS || CIFS=y && INFINIBAND=y && INFINIBAND_ADDR_TRANS=y help - Enables SMB Direct experimental support for SMB 3.0, 3.02 and 3.1.1. + Enables SMB Direct support for SMB 3.0, 3.02 and 3.1.1. SMB Direct allows transferring SMB packets over RDMA. If unsure, say N. diff --git a/fs/cifs/Makefile b/fs/cifs/Makefile index 51af69a1a328..41332f20055b 100644 --- a/fs/cifs/Makefile +++ b/fs/cifs/Makefile @@ -10,10 +10,9 @@ cifs-y := trace.o cifsfs.o cifssmb.o cifs_debug.o connect.o dir.o file.o \ cifs_unicode.o nterr.o cifsencrypt.o \ readdir.o ioctl.o sess.o export.o smb1ops.o winucase.o \ smb2ops.o smb2maperror.o smb2transport.o \ - smb2misc.o smb2pdu.o smb2inode.o smb2file.o + smb2misc.o smb2pdu.o smb2inode.o smb2file.o cifsacl.o cifs-$(CONFIG_CIFS_XATTR) += xattr.o -cifs-$(CONFIG_CIFS_ACL) += cifsacl.o cifs-$(CONFIG_CIFS_UPCALL) += cifs_spnego.o diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index ec933fb0b36e..a38d796f5ffe 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -240,9 +240,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) #ifdef CONFIG_CIFS_XATTR seq_printf(m, ",XATTR"); #endif -#ifdef CONFIG_CIFS_ACL seq_printf(m, ",ACL"); -#endif seq_putc(m, '\n'); seq_printf(m, "CIFSMaxBufSize: %d\n", CIFSMaxBufSize); seq_printf(m, "Active VFS Requests: %d\n", GlobalTotalActiveXid); diff --git a/fs/cifs/cifs_fs_sb.h b/fs/cifs/cifs_fs_sb.h index ed49222abecb..b326d2ca3765 100644 --- a/fs/cifs/cifs_fs_sb.h +++ b/fs/cifs/cifs_fs_sb.h @@ -52,6 +52,7 @@ #define CIFS_MOUNT_UID_FROM_ACL 0x2000000 /* try to get UID via special SID */ #define CIFS_MOUNT_NO_HANDLE_CACHE 0x4000000 /* disable caching dir handles */ #define CIFS_MOUNT_NO_DFS 0x8000000 /* disable DFS resolving */ +#define CIFS_MOUNT_MODE_FROM_SID 0x10000000 /* retrieve mode from special ACE */ struct cifs_sb_info { struct rb_root tlink_tree; @@ -83,5 +84,10 @@ struct cifs_sb_info { * failover properly. */ char *origin_fullpath; /* \\HOST\SHARE\[OPTIONAL PATH] */ + /* + * Indicate whether serverino option was turned off later + * (cifs_autodisable_serverino) in order to match new mounts. + */ + bool mnt_cifs_serverino_autodisabled; }; #endif /* _CIFS_FS_SB_H */ diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index 24635b65effa..270d3c58fb3b 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -526,6 +526,8 @@ cifs_show_options(struct seq_file *s, struct dentry *root) seq_puts(s, ",nobrl"); if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_HANDLE_CACHE) seq_puts(s, ",nohandlecache"); + if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) + seq_puts(s, ",modefromsid"); if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) seq_puts(s, ",cifsacl"); if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM) @@ -554,6 +556,11 @@ cifs_show_options(struct seq_file *s, struct dentry *root) seq_printf(s, ",bsize=%u", cifs_sb->bsize); seq_printf(s, ",echo_interval=%lu", tcon->ses->server->echo_interval / HZ); + + /* Only display max_credits if it was overridden on mount */ + if (tcon->ses->server->max_credits != SMB2_MAX_CREDITS_AVAILABLE) + seq_printf(s, ",max_credits=%u", tcon->ses->server->max_credits); + if (tcon->snapshot_time) seq_printf(s, ",snapshot=%llu", tcon->snapshot_time); if (tcon->handle_timeout) @@ -1517,11 +1524,9 @@ init_cifs(void) goto out_destroy_dfs_cache; #endif /* CONFIG_CIFS_UPCALL */ -#ifdef CONFIG_CIFS_ACL rc = init_cifs_idmap(); if (rc) goto out_register_key_type; -#endif /* CONFIG_CIFS_ACL */ rc = register_filesystem(&cifs_fs_type); if (rc) @@ -1536,10 +1541,8 @@ init_cifs(void) return 0; out_init_cifs_idmap: -#ifdef CONFIG_CIFS_ACL exit_cifs_idmap(); out_register_key_type: -#endif #ifdef CONFIG_CIFS_UPCALL exit_cifs_spnego(); out_destroy_dfs_cache: @@ -1571,9 +1574,7 @@ exit_cifs(void) unregister_filesystem(&cifs_fs_type); unregister_filesystem(&smb3_fs_type); cifs_dfs_release_automount_timer(); -#ifdef CONFIG_CIFS_ACL exit_cifs_idmap(); -#endif #ifdef CONFIG_CIFS_UPCALL exit_cifs_spnego(); #endif @@ -1607,5 +1608,6 @@ MODULE_SOFTDEP("pre: sha256"); MODULE_SOFTDEP("pre: sha512"); MODULE_SOFTDEP("pre: aead2"); MODULE_SOFTDEP("pre: ccm"); +MODULE_SOFTDEP("pre: gcm"); module_init(init_cifs) module_exit(exit_cifs) diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index 4777b3c4a92c..fe610e7e3670 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -550,6 +550,7 @@ struct smb_vol { bool override_gid:1; bool dynperm:1; bool noperm:1; + bool mode_ace:1; bool no_psx_acl:1; /* set if posix acl support should be disabled */ bool cifs_acl:1; bool backupuid_specified; /* mount option backupuid is specified */ @@ -600,6 +601,7 @@ struct smb_vol { __u64 snapshot_time; /* needed for timewarp tokens */ __u32 handle_timeout; /* persistent and durable handle timeout in ms */ unsigned int max_credits; /* smb3 max_credits 10 < credits < 60000 */ + __u16 compression; /* compression algorithm 0xFFFF default 0=disabled */ }; /** @@ -617,7 +619,8 @@ struct smb_vol { CIFS_MOUNT_FSCACHE | CIFS_MOUNT_MF_SYMLINKS | \ CIFS_MOUNT_MULTIUSER | CIFS_MOUNT_STRICT_IO | \ CIFS_MOUNT_CIFS_BACKUPUID | CIFS_MOUNT_CIFS_BACKUPGID | \ - CIFS_MOUNT_NO_DFS) + CIFS_MOUNT_UID_FROM_ACL | CIFS_MOUNT_NO_HANDLE_CACHE | \ + CIFS_MOUNT_NO_DFS | CIFS_MOUNT_MODE_FROM_SID) /** * Generic VFS superblock mount flags (s_flags) to consider when @@ -1870,7 +1873,6 @@ extern unsigned int cifs_min_small; /* min size of small buf pool */ extern unsigned int cifs_max_pending; /* MAX requests at once to server*/ extern bool disable_legacy_dialects; /* forbid vers=1.0 and vers=2.0 mounts */ -#ifdef CONFIG_CIFS_ACL GLOBAL_EXTERN struct rb_root uidtree; GLOBAL_EXTERN struct rb_root gidtree; GLOBAL_EXTERN spinlock_t siduidlock; @@ -1879,7 +1881,6 @@ GLOBAL_EXTERN struct rb_root siduidtree; GLOBAL_EXTERN struct rb_root sidgidtree; GLOBAL_EXTERN spinlock_t uidsidlock; GLOBAL_EXTERN spinlock_t gidsidlock; -#endif /* CONFIG_CIFS_ACL */ void cifs_oplock_break(struct work_struct *work); void cifs_queue_oplock_break(struct cifsFileInfo *cfile); diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c index 1fbd92843a73..e2f95965065d 100644 --- a/fs/cifs/cifssmb.c +++ b/fs/cifs/cifssmb.c @@ -3600,11 +3600,9 @@ static int cifs_copy_posix_acl(char *trgt, char *src, const int buflen, return size; } -static __u16 convert_ace_to_cifs_ace(struct cifs_posix_ace *cifs_ace, +static void convert_ace_to_cifs_ace(struct cifs_posix_ace *cifs_ace, const struct posix_acl_xattr_entry *local_ace) { - __u16 rc = 0; /* 0 = ACL converted ok */ - cifs_ace->cifs_e_perm = le16_to_cpu(local_ace->e_perm); cifs_ace->cifs_e_tag = le16_to_cpu(local_ace->e_tag); /* BB is there a better way to handle the large uid? */ @@ -3617,7 +3615,6 @@ static __u16 convert_ace_to_cifs_ace(struct cifs_posix_ace *cifs_ace, cifs_dbg(FYI, "perm %d tag %d id %d\n", ace->e_perm, ace->e_tag, ace->e_id); */ - return rc; } /* Convert ACL from local Linux POSIX xattr to CIFS POSIX ACL wire format */ @@ -3653,13 +3650,8 @@ static __u16 ACL_to_cifs_posix(char *parm_data, const char *pACL, cifs_dbg(FYI, "unknown ACL type %d\n", acl_type); return 0; } - for (i = 0; i < count; i++) { - rc = convert_ace_to_cifs_ace(&cifs_acl->ace_array[i], &ace[i]); - if (rc != 0) { - /* ACE not converted */ - break; - } - } + for (i = 0; i < count; i++) + convert_ace_to_cifs_ace(&cifs_acl->ace_array[i], &ace[i]); if (rc == 0) { rc = (__u16)(count * sizeof(struct cifs_posix_ace)); rc += sizeof(struct cifs_posix_acl); @@ -3920,7 +3912,6 @@ GetExtAttrOut: #endif /* CONFIG_POSIX */ -#ifdef CONFIG_CIFS_ACL /* * Initialize NT TRANSACT SMB into small smb request buffer. This assumes that * all NT TRANSACTS that we init here have total parm and data under about 400 @@ -4164,7 +4155,6 @@ setCifsAclRetry: return (rc); } -#endif /* CONFIG_CIFS_ACL */ /* Legacy Query Path Information call for lookup to old servers such as Win9x/WinME */ diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 714a359c7c8d..a4830ced0f98 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -96,7 +96,8 @@ enum { Opt_multiuser, Opt_sloppy, Opt_nosharesock, Opt_persistent, Opt_nopersistent, Opt_resilient, Opt_noresilient, - Opt_domainauto, Opt_rdma, + Opt_domainauto, Opt_rdma, Opt_modesid, + Opt_compress, /* Mount options which take numeric value */ Opt_backupuid, Opt_backupgid, Opt_uid, @@ -175,6 +176,7 @@ static const match_table_t cifs_mount_option_tokens = { { Opt_serverino, "serverino" }, { Opt_noserverino, "noserverino" }, { Opt_rwpidforward, "rwpidforward" }, + { Opt_modesid, "modefromsid" }, { Opt_cifsacl, "cifsacl" }, { Opt_nocifsacl, "nocifsacl" }, { Opt_acl, "acl" }, @@ -212,6 +214,7 @@ static const match_table_t cifs_mount_option_tokens = { { Opt_echo_interval, "echo_interval=%s" }, { Opt_max_credits, "max_credits=%s" }, { Opt_snapshot, "snapshot=%s" }, + { Opt_compress, "compress=%s" }, { Opt_blank_user, "user=" }, { Opt_blank_user, "username=" }, @@ -706,10 +709,10 @@ static bool server_unresponsive(struct TCP_Server_Info *server) { /* - * We need to wait 2 echo intervals to make sure we handle such + * We need to wait 3 echo intervals to make sure we handle such * situations right: * 1s client sends a normal SMB request - * 2s client gets a response + * 3s client gets a response * 30s echo workqueue job pops, and decides we got a response recently * and don't need to send another * ... @@ -718,9 +721,9 @@ server_unresponsive(struct TCP_Server_Info *server) */ if ((server->tcpStatus == CifsGood || server->tcpStatus == CifsNeedNegotiate) && - time_after(jiffies, server->lstrp + 2 * server->echo_interval)) { + time_after(jiffies, server->lstrp + 3 * server->echo_interval)) { cifs_dbg(VFS, "Server %s has not responded in %lu seconds. Reconnecting...\n", - server->hostname, (2 * server->echo_interval) / HZ); + server->hostname, (3 * server->echo_interval) / HZ); cifs_reconnect(server); wake_up(&server->response_q); return true; @@ -1223,11 +1226,11 @@ next_pdu: atomic_read(&midCount)); cifs_dump_mem("Received Data is: ", bufs[i], HEADER_SIZE(server)); + smb2_add_credits_from_hdr(bufs[i], server); #ifdef CONFIG_CIFS_DEBUG2 if (server->ops->dump_detail) server->ops->dump_detail(bufs[i], server); - smb2_add_credits_from_hdr(bufs[i], server); cifs_dump_mids(server); #endif /* CIFS_DEBUG2 */ } @@ -1830,6 +1833,9 @@ cifs_parse_mount_options(const char *mountdata, const char *devname, case Opt_rwpidforward: vol->rwpidforward = 1; break; + case Opt_modesid: + vol->mode_ace = 1; + break; case Opt_cifsacl: vol->cifs_acl = 1; break; @@ -1911,6 +1917,11 @@ cifs_parse_mount_options(const char *mountdata, const char *devname, case Opt_rdma: vol->rdma = true; break; + case Opt_compress: + vol->compression = UNKNOWN_TYPE; + cifs_dbg(VFS, + "SMB3 compression support is experimental\n"); + break; /* Numeric Values */ case Opt_backupuid: @@ -2544,8 +2555,15 @@ static int match_server(struct TCP_Server_Info *server, struct smb_vol *vol) if (vol->nosharesock) return 0; - /* BB update this for smb3any and default case */ - if ((server->vals != vol->vals) || (server->ops != vol->ops)) + /* If multidialect negotiation see if existing sessions match one */ + if (strcmp(vol->vals->version_string, SMB3ANY_VERSION_STRING) == 0) { + if (server->vals->protocol_id < SMB30_PROT_ID) + return 0; + } else if (strcmp(vol->vals->version_string, + SMBDEFAULT_VERSION_STRING) == 0) { + if (server->vals->protocol_id < SMB21_PROT_ID) + return 0; + } else if ((server->vals != vol->vals) || (server->ops != vol->ops)) return 0; if (!net_eq(cifs_net_ns(server), current->nsproxy->net_ns)) @@ -2680,6 +2698,7 @@ cifs_get_tcp_session(struct smb_vol *volume_info) tcp_ses->sequence_number = 0; tcp_ses->reconnect_instance = 1; tcp_ses->lstrp = jiffies; + tcp_ses->compress_algorithm = cpu_to_le16(volume_info->compression); spin_lock_init(&tcp_ses->req_lock); INIT_LIST_HEAD(&tcp_ses->tcp_ses_list); INIT_LIST_HEAD(&tcp_ses->smb_ses_list); @@ -3460,12 +3479,16 @@ compare_mount_options(struct super_block *sb, struct cifs_mnt_data *mnt_data) { struct cifs_sb_info *old = CIFS_SB(sb); struct cifs_sb_info *new = mnt_data->cifs_sb; + unsigned int oldflags = old->mnt_cifs_flags & CIFS_MOUNT_MASK; + unsigned int newflags = new->mnt_cifs_flags & CIFS_MOUNT_MASK; if ((sb->s_flags & CIFS_MS_MASK) != (mnt_data->flags & CIFS_MS_MASK)) return 0; - if ((old->mnt_cifs_flags & CIFS_MOUNT_MASK) != - (new->mnt_cifs_flags & CIFS_MOUNT_MASK)) + if (old->mnt_cifs_serverino_autodisabled) + newflags &= ~CIFS_MOUNT_SERVER_INUM; + + if (oldflags != newflags) return 0; /* @@ -3965,6 +3988,8 @@ int cifs_setup_cifs_sb(struct smb_vol *pvolume_info, cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NOPOSIXBRL; if (pvolume_info->rwpidforward) cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_RWPIDFORWARD; + if (pvolume_info->mode_ace) + cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_MODE_FROM_SID; if (pvolume_info->cifs_acl) cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_CIFS_ACL; if (pvolume_info->backupuid_specified) { @@ -4459,11 +4484,13 @@ cifs_are_all_path_components_accessible(struct TCP_Server_Info *server, unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, - char *full_path) + char *full_path, + int added_treename) { int rc; char *s; char sep, tmp; + int skip = added_treename ? 1 : 0; sep = CIFS_DIR_SEP(cifs_sb); s = full_path; @@ -4478,7 +4505,14 @@ cifs_are_all_path_components_accessible(struct TCP_Server_Info *server, /* next separator */ while (*s && *s != sep) s++; - + /* + * if the treename is added, we then have to skip the first + * part within the separators + */ + if (skip) { + skip = 0; + continue; + } /* * temporarily null-terminate the path at the end of * the current component @@ -4526,8 +4560,7 @@ static int is_path_remote(struct cifs_sb_info *cifs_sb, struct smb_vol *vol, if (rc != -EREMOTE) { rc = cifs_are_all_path_components_accessible(server, xid, tcon, - cifs_sb, - full_path); + cifs_sb, full_path, tcon->Flags & SMB_SHARE_IS_IN_DFS); if (rc != 0) { cifs_dbg(VFS, "cannot query dirs between root and final path, " "enabling CIFS_MOUNT_USE_PREFIX_PATH\n"); diff --git a/fs/cifs/dfs_cache.c b/fs/cifs/dfs_cache.c index e3e1c13df439..1692c0c6c23a 100644 --- a/fs/cifs/dfs_cache.c +++ b/fs/cifs/dfs_cache.c @@ -492,7 +492,7 @@ static struct dfs_cache_entry *__find_cache_entry(unsigned int hash, #ifdef CONFIG_CIFS_DEBUG2 char *name = get_tgt_name(ce); - if (unlikely(IS_ERR(name))) { + if (IS_ERR(name)) { rcu_read_unlock(); return ERR_CAST(name); } diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index d7cc62252634..1bffe029fb66 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -892,7 +892,6 @@ cifs_get_inode_info(struct inode **inode, const char *full_path, cifs_dbg(FYI, "cifs_sfu_type failed: %d\n", tmprc); } -#ifdef CONFIG_CIFS_ACL /* fill in 0777 bits from ACL */ if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) { rc = cifs_acl_to_fattr(cifs_sb, &fattr, *inode, full_path, fid); @@ -902,7 +901,6 @@ cifs_get_inode_info(struct inode **inode, const char *full_path, goto cgii_exit; } } -#endif /* CONFIG_CIFS_ACL */ /* fill in remaining high mode bits e.g. SUID, VTX */ if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL) @@ -2415,7 +2413,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) xid = get_xid(); - cifs_dbg(FYI, "setattr on file %pd attrs->iavalid 0x%x\n", + cifs_dbg(FYI, "setattr on file %pd attrs->ia_valid 0x%x\n", direntry, attrs->ia_valid); if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_PERM) @@ -2466,7 +2464,6 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (attrs->ia_valid & ATTR_GID) gid = attrs->ia_gid; -#ifdef CONFIG_CIFS_ACL if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) { if (uid_valid(uid) || gid_valid(gid)) { rc = id_mode_to_cifs_acl(inode, full_path, NO_CHANGE_64, @@ -2478,7 +2475,6 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) } } } else -#endif /* CONFIG_CIFS_ACL */ if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID)) attrs->ia_valid &= ~(ATTR_UID | ATTR_GID); @@ -2489,7 +2485,6 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (attrs->ia_valid & ATTR_MODE) { mode = attrs->ia_mode; rc = 0; -#ifdef CONFIG_CIFS_ACL if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) { rc = id_mode_to_cifs_acl(inode, full_path, mode, INVALID_UID, INVALID_GID); @@ -2499,7 +2494,6 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) goto cifs_setattr_exit; } } else -#endif /* CONFIG_CIFS_ACL */ if (((mode & S_IWUGO) == 0) && (cifsInode->cifsAttrs & ATTR_READONLY) == 0) { diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c index b1a696a73f7c..f383877a6511 100644 --- a/fs/cifs/misc.c +++ b/fs/cifs/misc.c @@ -539,6 +539,7 @@ cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb) tcon = cifs_sb_master_tcon(cifs_sb); cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_SERVER_INUM; + cifs_sb->mnt_cifs_serverino_autodisabled = true; cifs_dbg(VFS, "Autodisabling the use of server inode numbers on %s.\n", tcon ? tcon->treeName : "new server"); cifs_dbg(VFS, "The server doesn't seem to support them properly or the files might be on different servers (DFS).\n"); diff --git a/fs/cifs/smb1ops.c b/fs/cifs/smb1ops.c index 9e430ae9314f..b7421a096319 100644 --- a/fs/cifs/smb1ops.c +++ b/fs/cifs/smb1ops.c @@ -1223,16 +1223,15 @@ struct smb_version_operations smb1_operations = { .query_all_EAs = CIFSSMBQAllEAs, .set_EA = CIFSSMBSetEA, #endif /* CIFS_XATTR */ -#ifdef CONFIG_CIFS_ACL .get_acl = get_cifs_acl, .get_acl_by_fid = get_cifs_acl_by_fid, .set_acl = set_cifs_acl, -#endif /* CIFS_ACL */ .make_node = cifs_make_node, }; struct smb_version_values smb1_values = { .version_string = SMB1_VERSION_STRING, + .protocol_id = SMB10_PROT_ID, .large_lock_type = LOCKING_ANDX_LARGE_FILES, .exclusive_lock_type = 0, .shared_lock_type = LOCKING_ANDX_SHARED_LOCK, diff --git a/fs/cifs/smb2inode.c b/fs/cifs/smb2inode.c index 278405d26c47..d8d9cdfa30b6 100644 --- a/fs/cifs/smb2inode.c +++ b/fs/cifs/smb2inode.c @@ -120,6 +120,8 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_O_INFO_FILE, 0, sizeof(struct smb2_file_all_info) + PATH_MAX * 2, 0, NULL); + if (rc) + goto finished; smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); trace_smb3_query_info_compound_enter(xid, ses->Suid, tcon->tid, @@ -147,6 +149,8 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, current->tgid, FILE_DISPOSITION_INFORMATION, SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto finished; smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); trace_smb3_rmdir_enter(xid, ses->Suid, tcon->tid, full_path); @@ -163,6 +167,8 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, current->tgid, FILE_END_OF_FILE_INFORMATION, SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto finished; smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); trace_smb3_set_eof_enter(xid, ses->Suid, tcon->tid, full_path); @@ -180,6 +186,8 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, current->tgid, FILE_BASIC_INFORMATION, SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto finished; smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); trace_smb3_set_info_compound_enter(xid, ses->Suid, tcon->tid, @@ -206,6 +214,8 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, current->tgid, FILE_RENAME_INFORMATION, SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto finished; smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); trace_smb3_rename_enter(xid, ses->Suid, tcon->tid, full_path); @@ -231,6 +241,8 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, current->tgid, FILE_LINK_INFORMATION, SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto finished; smb2_set_next_command(tcon, &rqst[num_rqst]); smb2_set_related(&rqst[num_rqst++]); trace_smb3_hardlink_enter(xid, ses->Suid, tcon->tid, full_path); diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 9fd56b0acd7e..0cdc4e47ca87 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -2027,6 +2027,10 @@ smb2_set_related(struct smb_rqst *rqst) struct smb2_sync_hdr *shdr; shdr = (struct smb2_sync_hdr *)(rqst->rq_iov[0].iov_base); + if (shdr == NULL) { + cifs_dbg(FYI, "shdr NULL in smb2_set_related\n"); + return; + } shdr->Flags |= SMB2_FLAGS_RELATED_OPERATIONS; } @@ -2041,6 +2045,12 @@ smb2_set_next_command(struct cifs_tcon *tcon, struct smb_rqst *rqst) unsigned long len = smb_rqst_len(server, rqst); int i, num_padding; + shdr = (struct smb2_sync_hdr *)(rqst->rq_iov[0].iov_base); + if (shdr == NULL) { + cifs_dbg(FYI, "shdr NULL in smb2_set_next_command\n"); + return; + } + /* SMB headers in a compound are 8 byte aligned. */ /* No padding needed */ @@ -2080,7 +2090,6 @@ smb2_set_next_command(struct cifs_tcon *tcon, struct smb_rqst *rqst) } finished: - shdr = (struct smb2_sync_hdr *)(rqst->rq_iov[0].iov_base); shdr->NextCommand = cpu_to_le32(len); } @@ -2374,6 +2383,34 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, } static int +parse_reparse_posix(struct reparse_posix_data *symlink_buf, + u32 plen, char **target_path, + struct cifs_sb_info *cifs_sb) +{ + unsigned int len; + + /* See MS-FSCC 2.1.2.6 for the 'NFS' style reparse tags */ + len = le16_to_cpu(symlink_buf->ReparseDataLength); + + if (le64_to_cpu(symlink_buf->InodeType) != NFS_SPECFILE_LNK) { + cifs_dbg(VFS, "%lld not a supported symlink type\n", + le64_to_cpu(symlink_buf->InodeType)); + return -EOPNOTSUPP; + } + + *target_path = cifs_strndup_from_utf16( + symlink_buf->PathBuffer, + len, true, cifs_sb->local_nls); + if (!(*target_path)) + return -ENOMEM; + + convert_delimiter(*target_path, '/'); + cifs_dbg(FYI, "%s: target path: %s\n", __func__, *target_path); + + return 0; +} + +static int parse_reparse_symlink(struct reparse_symlink_data_buffer *symlink_buf, u32 plen, char **target_path, struct cifs_sb_info *cifs_sb) @@ -2381,11 +2418,7 @@ parse_reparse_symlink(struct reparse_symlink_data_buffer *symlink_buf, unsigned int sub_len; unsigned int sub_offset; - /* We only handle Symbolic Link : MS-FSCC 2.1.2.4 */ - if (le32_to_cpu(symlink_buf->ReparseTag) != IO_REPARSE_TAG_SYMLINK) { - cifs_dbg(VFS, "srv returned invalid symlink buffer\n"); - return -EIO; - } + /* We handle Symbolic Link reparse tag here. See: MS-FSCC 2.1.2.4 */ sub_offset = le16_to_cpu(symlink_buf->SubstituteNameOffset); sub_len = le16_to_cpu(symlink_buf->SubstituteNameLength); @@ -2407,6 +2440,41 @@ parse_reparse_symlink(struct reparse_symlink_data_buffer *symlink_buf, return 0; } +static int +parse_reparse_point(struct reparse_data_buffer *buf, + u32 plen, char **target_path, + struct cifs_sb_info *cifs_sb) +{ + if (plen < sizeof(struct reparse_data_buffer)) { + cifs_dbg(VFS, "reparse buffer is too small. Must be " + "at least 8 bytes but was %d\n", plen); + return -EIO; + } + + if (plen < le16_to_cpu(buf->ReparseDataLength) + + sizeof(struct reparse_data_buffer)) { + cifs_dbg(VFS, "srv returned invalid reparse buf " + "length: %d\n", plen); + return -EIO; + } + + /* See MS-FSCC 2.1.2 */ + switch (le32_to_cpu(buf->ReparseTag)) { + case IO_REPARSE_TAG_NFS: + return parse_reparse_posix( + (struct reparse_posix_data *)buf, + plen, target_path, cifs_sb); + case IO_REPARSE_TAG_SYMLINK: + return parse_reparse_symlink( + (struct reparse_symlink_data_buffer *)buf, + plen, target_path, cifs_sb); + default: + cifs_dbg(VFS, "srv returned unknown symlink buffer " + "tag:0x%08x\n", le32_to_cpu(buf->ReparseTag)); + return -EOPNOTSUPP; + } +} + #define SMB2_SYMLINK_STRUCT_SIZE \ (sizeof(struct smb2_err_rsp) - 1 + sizeof(struct smb2_symlink_err_rsp)) @@ -2533,23 +2601,8 @@ smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon, goto querty_exit; } - if (plen < 8) { - cifs_dbg(VFS, "reparse buffer is too small. Must be " - "at least 8 bytes but was %d\n", plen); - rc = -EIO; - goto querty_exit; - } - - if (plen < le16_to_cpu(reparse_buf->ReparseDataLength) + 8) { - cifs_dbg(VFS, "srv returned invalid reparse buf " - "length: %d\n", plen); - rc = -EIO; - goto querty_exit; - } - - rc = parse_reparse_symlink( - (struct reparse_symlink_data_buffer *)reparse_buf, - plen, target_path, cifs_sb); + rc = parse_reparse_point(reparse_buf, plen, target_path, + cifs_sb); goto querty_exit; } @@ -2561,26 +2614,32 @@ smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon, err_buf = err_iov.iov_base; if (le32_to_cpu(err_buf->ByteCount) < sizeof(struct smb2_symlink_err_rsp) || err_iov.iov_len < SMB2_SYMLINK_STRUCT_SIZE) { - rc = -ENOENT; + rc = -EINVAL; + goto querty_exit; + } + + symlink = (struct smb2_symlink_err_rsp *)err_buf->ErrorData; + if (le32_to_cpu(symlink->SymLinkErrorTag) != SYMLINK_ERROR_TAG || + le32_to_cpu(symlink->ReparseTag) != IO_REPARSE_TAG_SYMLINK) { + rc = -EINVAL; goto querty_exit; } /* open must fail on symlink - reset rc */ rc = 0; - symlink = (struct smb2_symlink_err_rsp *)err_buf->ErrorData; sub_len = le16_to_cpu(symlink->SubstituteNameLength); sub_offset = le16_to_cpu(symlink->SubstituteNameOffset); print_len = le16_to_cpu(symlink->PrintNameLength); print_offset = le16_to_cpu(symlink->PrintNameOffset); if (err_iov.iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offset + sub_len) { - rc = -ENOENT; + rc = -EINVAL; goto querty_exit; } if (err_iov.iov_len < SMB2_SYMLINK_STRUCT_SIZE + print_offset + print_len) { - rc = -ENOENT; + rc = -EINVAL; goto querty_exit; } @@ -2606,7 +2665,6 @@ smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon, return rc; } -#ifdef CONFIG_CIFS_ACL static struct cifs_ntsd * get_smb2_acl_by_fid(struct cifs_sb_info *cifs_sb, const struct cifs_fid *cifsfid, u32 *pacllen) @@ -2691,7 +2749,6 @@ get_smb2_acl_by_path(struct cifs_sb_info *cifs_sb, return pntsd; } -#ifdef CONFIG_CIFS_ACL static int set_smb2_acl(struct cifs_ntsd *pnntsd, __u32 acllen, struct inode *inode, const char *path, int aclflag) @@ -2749,7 +2806,6 @@ set_smb2_acl(struct cifs_ntsd *pnntsd, __u32 acllen, free_xid(xid); return rc; } -#endif /* CIFS_ACL */ /* Retrieve an ACL from the server */ static struct cifs_ntsd * @@ -2769,7 +2825,6 @@ get_smb2_acl(struct cifs_sb_info *cifs_sb, cifsFileInfo_put(open_file); return pntsd; } -#endif static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, loff_t offset, loff_t len, bool keep_size) @@ -3367,7 +3422,7 @@ smb2_dir_needs_close(struct cifsFileInfo *cfile) static void fill_transform_hdr(struct smb2_transform_hdr *tr_hdr, unsigned int orig_len, - struct smb_rqst *old_rq) + struct smb_rqst *old_rq, __le16 cipher_type) { struct smb2_sync_hdr *shdr = (struct smb2_sync_hdr *)old_rq->rq_iov[0].iov_base; @@ -3376,7 +3431,10 @@ fill_transform_hdr(struct smb2_transform_hdr *tr_hdr, unsigned int orig_len, tr_hdr->ProtocolId = SMB2_TRANSFORM_PROTO_NUM; tr_hdr->OriginalMessageSize = cpu_to_le32(orig_len); tr_hdr->Flags = cpu_to_le16(0x01); - get_random_bytes(&tr_hdr->Nonce, SMB3_AES128CMM_NONCE); + if (cipher_type == SMB2_ENCRYPTION_AES128_GCM) + get_random_bytes(&tr_hdr->Nonce, SMB3_AES128GCM_NONCE); + else + get_random_bytes(&tr_hdr->Nonce, SMB3_AES128CCM_NONCE); memcpy(&tr_hdr->SessionId, &shdr->SessionId, 8); } @@ -3534,8 +3592,13 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, rc = -ENOMEM; goto free_sg; } - iv[0] = 3; - memcpy(iv + 1, (char *)tr_hdr->Nonce, SMB3_AES128CMM_NONCE); + + if (server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) + memcpy(iv, (char *)tr_hdr->Nonce, SMB3_AES128GCM_NONCE); + else { + iv[0] = 3; + memcpy(iv + 1, (char *)tr_hdr->Nonce, SMB3_AES128CCM_NONCE); + } aead_request_set_crypt(req, sg, sg, crypt_len, iv); aead_request_set_ad(req, assoc_data_len); @@ -3635,7 +3698,7 @@ smb3_init_transform_rq(struct TCP_Server_Info *server, int num_rqst, } /* fill the 1st iov with a transform header */ - fill_transform_hdr(tr_hdr, orig_len, old_rq); + fill_transform_hdr(tr_hdr, orig_len, old_rq, server->cipher_type); rc = crypt_message(server, num_rqst, new_rq, 1); cifs_dbg(FYI, "Encrypt message returned %d\n", rc); @@ -4284,11 +4347,9 @@ struct smb_version_operations smb20_operations = { .query_all_EAs = smb2_query_eas, .set_EA = smb2_set_ea, #endif /* CIFS_XATTR */ -#ifdef CONFIG_CIFS_ACL .get_acl = get_smb2_acl, .get_acl_by_fid = get_smb2_acl_by_fid, .set_acl = set_smb2_acl, -#endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, .make_node = smb2_make_node, @@ -4385,11 +4446,9 @@ struct smb_version_operations smb21_operations = { .query_all_EAs = smb2_query_eas, .set_EA = smb2_set_ea, #endif /* CIFS_XATTR */ -#ifdef CONFIG_CIFS_ACL .get_acl = get_smb2_acl, .get_acl_by_fid = get_smb2_acl_by_fid, .set_acl = set_smb2_acl, -#endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, .make_node = smb2_make_node, @@ -4495,11 +4554,9 @@ struct smb_version_operations smb30_operations = { .query_all_EAs = smb2_query_eas, .set_EA = smb2_set_ea, #endif /* CIFS_XATTR */ -#ifdef CONFIG_CIFS_ACL .get_acl = get_smb2_acl, .get_acl_by_fid = get_smb2_acl_by_fid, .set_acl = set_smb2_acl, -#endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, .make_node = smb2_make_node, @@ -4606,11 +4663,9 @@ struct smb_version_operations smb311_operations = { .query_all_EAs = smb2_query_eas, .set_EA = smb2_set_ea, #endif /* CIFS_XATTR */ -#ifdef CONFIG_CIFS_ACL .get_acl = get_smb2_acl, .get_acl_by_fid = get_smb2_acl_by_fid, .set_acl = set_smb2_acl, -#endif /* CIFS_ACL */ .next_header = smb2_next_header, .ioctl_query_info = smb2_ioctl_query_info, .make_node = smb2_make_node, diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 75311a8a68bf..f58e4dc3987b 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -489,10 +489,25 @@ static void build_encrypt_ctxt(struct smb2_encryption_neg_context *pneg_ctxt) { pneg_ctxt->ContextType = SMB2_ENCRYPTION_CAPABILITIES; - pneg_ctxt->DataLength = cpu_to_le16(4); /* Cipher Count + le16 cipher */ - pneg_ctxt->CipherCount = cpu_to_le16(1); -/* pneg_ctxt->Ciphers[0] = SMB2_ENCRYPTION_AES128_GCM;*/ /* not supported yet */ - pneg_ctxt->Ciphers[0] = SMB2_ENCRYPTION_AES128_CCM; + pneg_ctxt->DataLength = cpu_to_le16(6); /* Cipher Count + two ciphers */ + pneg_ctxt->CipherCount = cpu_to_le16(2); + pneg_ctxt->Ciphers[0] = SMB2_ENCRYPTION_AES128_GCM; + pneg_ctxt->Ciphers[1] = SMB2_ENCRYPTION_AES128_CCM; +} + +static unsigned int +build_netname_ctxt(struct smb2_netname_neg_context *pneg_ctxt, char *hostname) +{ + struct nls_table *cp = load_nls_default(); + + pneg_ctxt->ContextType = SMB2_NETNAME_NEGOTIATE_CONTEXT_ID; + + /* copy up to max of first 100 bytes of server name to NetName field */ + pneg_ctxt->DataLength = cpu_to_le16(2 + + (2 * cifs_strtoUTF16(pneg_ctxt->NetName, hostname, 100, cp))); + /* context size is DataLength + minimal smb2_neg_context */ + return DIV_ROUND_UP(le16_to_cpu(pneg_ctxt->DataLength) + + sizeof(struct smb2_neg_context), 8) * 8; } static void @@ -521,7 +536,7 @@ build_posix_ctxt(struct smb2_posix_neg_context *pneg_ctxt) static void assemble_neg_contexts(struct smb2_negotiate_req *req, - unsigned int *total_len) + struct TCP_Server_Info *server, unsigned int *total_len) { char *pneg_ctxt = (char *)req; unsigned int ctxt_len; @@ -551,17 +566,25 @@ assemble_neg_contexts(struct smb2_negotiate_req *req, *total_len += ctxt_len; pneg_ctxt += ctxt_len; - build_compression_ctxt((struct smb2_compression_capabilities_context *) + if (server->compress_algorithm) { + build_compression_ctxt((struct smb2_compression_capabilities_context *) pneg_ctxt); - ctxt_len = DIV_ROUND_UP( - sizeof(struct smb2_compression_capabilities_context), 8) * 8; + ctxt_len = DIV_ROUND_UP( + sizeof(struct smb2_compression_capabilities_context), + 8) * 8; + *total_len += ctxt_len; + pneg_ctxt += ctxt_len; + req->NegotiateContextCount = cpu_to_le16(5); + } else + req->NegotiateContextCount = cpu_to_le16(4); + + ctxt_len = build_netname_ctxt((struct smb2_netname_neg_context *)pneg_ctxt, + server->hostname); *total_len += ctxt_len; pneg_ctxt += ctxt_len; build_posix_ctxt((struct smb2_posix_neg_context *)pneg_ctxt); *total_len += sizeof(struct smb2_posix_neg_context); - - req->NegotiateContextCount = cpu_to_le16(4); } static void decode_preauth_context(struct smb2_preauth_neg_context *ctxt) @@ -829,7 +852,7 @@ SMB2_negotiate(const unsigned int xid, struct cifs_ses *ses) if ((ses->server->vals->protocol_id == SMB311_PROT_ID) || (strcmp(ses->server->vals->version_string, SMBDEFAULT_VERSION_STRING) == 0)) - assemble_neg_contexts(req, &total_len); + assemble_neg_contexts(req, server, &total_len); } iov[0].iov_base = (char *)req; iov[0].iov_len = total_len; @@ -2095,6 +2118,48 @@ add_twarp_context(struct kvec *iov, unsigned int *num_iovec, __u64 timewarp) return 0; } +static struct crt_query_id_ctxt * +create_query_id_buf(void) +{ + struct crt_query_id_ctxt *buf; + + buf = kzalloc(sizeof(struct crt_query_id_ctxt), GFP_KERNEL); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = cpu_to_le16(0); + buf->ccontext.DataLength = cpu_to_le32(0); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct crt_query_id_ctxt, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_QUERY_ON_DISK_ID is "QFid" */ + buf->Name[0] = 'Q'; + buf->Name[1] = 'F'; + buf->Name[2] = 'i'; + buf->Name[3] = 'd'; + return buf; +} + +/* See MS-SMB2 2.2.13.2.9 */ +static int +add_query_id_context(struct kvec *iov, unsigned int *num_iovec) +{ + struct smb2_create_req *req = iov[0].iov_base; + unsigned int num = *num_iovec; + + iov[num].iov_base = create_query_id_buf(); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = sizeof(struct crt_query_id_ctxt); + if (!req->CreateContextsOffset) + req->CreateContextsOffset = cpu_to_le32( + sizeof(struct smb2_create_req) + + iov[num - 1].iov_len); + le32_add_cpu(&req->CreateContextsLength, sizeof(struct crt_query_id_ctxt)); + *num_iovec = num + 1; + return 0; +} + static int alloc_path_with_tree_prefix(__le16 **out_path, int *out_size, int *out_len, const char *treename, const __le16 *path) @@ -2423,6 +2488,12 @@ SMB2_open_init(struct cifs_tcon *tcon, struct smb_rqst *rqst, __u8 *oplock, return rc; } + if (n_iov > 2) { + struct create_context *ccontext = + (struct create_context *)iov[n_iov-1].iov_base; + ccontext->Next = cpu_to_le32(iov[n_iov-1].iov_len); + } + add_query_id_context(iov, &n_iov); rqst->rq_nvec = n_iov; return 0; @@ -2550,12 +2621,11 @@ SMB2_ioctl_init(struct cifs_tcon *tcon, struct smb_rqst *rqst, * indatalen is usually small at a couple of bytes max, so * just allocate through generic pool */ - in_data_buf = kmalloc(indatalen, GFP_NOFS); + in_data_buf = kmemdup(in_data, indatalen, GFP_NOFS); if (!in_data_buf) { cifs_small_buf_release(req); return -ENOMEM; } - memcpy(in_data_buf, in_data, indatalen); } req->CtlCode = cpu_to_le32(opcode); diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h index 858353d20c39..7e2e782f8edd 100644 --- a/fs/cifs/smb2pdu.h +++ b/fs/cifs/smb2pdu.h @@ -123,7 +123,7 @@ struct smb2_sync_pdu { __le16 StructureSize2; /* size of wct area (varies, request specific) */ } __packed; -#define SMB3_AES128CMM_NONCE 11 +#define SMB3_AES128CCM_NONCE 11 #define SMB3_AES128GCM_NONCE 12 struct smb2_transform_hdr { @@ -166,6 +166,8 @@ struct smb2_err_rsp { __u8 ErrorData[1]; /* variable length */ } __packed; +#define SYMLINK_ERROR_TAG 0x4c4d5953 + struct smb2_symlink_err_rsp { __le32 SymLinkLength; __le32 SymLinkErrorTag; @@ -227,6 +229,7 @@ struct smb2_negotiate_req { } __packed; /* Dialects */ +#define SMB10_PROT_ID 0x0000 /* local only, not sent on wire w/CIFS negprot */ #define SMB20_PROT_ID 0x0202 #define SMB21_PROT_ID 0x0210 #define SMB30_PROT_ID 0x0300 @@ -293,7 +296,7 @@ struct smb2_encryption_neg_context { __le16 DataLength; __le32 Reserved; __le16 CipherCount; /* AES-128-GCM and AES-128-CCM */ - __le16 Ciphers[1]; /* Ciphers[0] since only one used now */ + __le16 Ciphers[2]; } __packed; /* See MS-SMB2 2.2.3.1.3 */ @@ -316,6 +319,12 @@ struct smb2_compression_capabilities_context { * For smb2_netname_negotiate_context_id See MS-SMB2 2.2.3.1.4. * Its struct simply contains NetName, an array of Unicode characters */ +struct smb2_netname_neg_context { + __le16 ContextType; /* 0x100 */ + __le16 DataLength; + __le32 Reserved; + __le16 NetName[0]; /* hostname of target converted to UCS-2 */ +} __packed; #define POSIX_CTXT_DATA_LEN 16 struct smb2_posix_neg_context { @@ -640,6 +649,7 @@ struct smb2_tree_disconnect_rsp { #define SMB2_CREATE_DURABLE_HANDLE_REQUEST_V2 "DH2Q" #define SMB2_CREATE_DURABLE_HANDLE_RECONNECT_V2 "DH2C" #define SMB2_CREATE_APP_INSTANCE_ID 0x45BCA66AEFA7F74A9008FA462E144D74 +#define SMB2_CREATE_APP_INSTANCE_VERSION 0xB982D0B73B56074FA07B524A8116A010 #define SVHDX_OPEN_DEVICE_CONTEX 0x9CCBCF9E04C1E643980E158DA1F6EC83 #define SMB2_CREATE_TAG_POSIX 0x93AD25509CB411E7B42383DE968BCD7C @@ -654,9 +664,10 @@ struct smb2_tree_disconnect_rsp { * [3] : durable context * [4] : posix context * [5] : time warp context - * [6] : compound padding + * [6] : query id context + * [7] : compound padding */ -#define SMB2_CREATE_IOV_SIZE 7 +#define SMB2_CREATE_IOV_SIZE 8 struct smb2_create_req { struct smb2_sync_hdr sync_hdr; @@ -680,10 +691,10 @@ struct smb2_create_req { /* * Maximum size of a SMB2_CREATE response is 64 (smb2 header) + - * 88 (fixed part of create response) + 520 (path) + 150 (contexts) + + * 88 (fixed part of create response) + 520 (path) + 208 (contexts) + * 2 bytes of padding. */ -#define MAX_SMB2_CREATE_RESPONSE_SIZE 824 +#define MAX_SMB2_CREATE_RESPONSE_SIZE 880 struct smb2_create_rsp { struct smb2_sync_hdr sync_hdr; @@ -806,6 +817,13 @@ struct durable_reconnect_context_v2 { __le32 Flags; /* see above DHANDLE_FLAG_PERSISTENT */ } __packed; +/* See MS-SMB2 2.2.14.2.9 */ +struct on_disk_id { + __le64 DiskFileId; + __le64 VolumeId; + __u32 Reserved[4]; +} __packed; + /* See MS-SMB2 2.2.14.2.12 */ struct durable_reconnect_context_v2_rsp { __le32 Timeout; @@ -826,6 +844,12 @@ struct crt_twarp_ctxt { } __packed; +/* See MS-SMB2 2.2.13.2.9 */ +struct crt_query_id_ctxt { + struct create_context ccontext; + __u8 Name[8]; +} __packed; + #define COPY_CHUNK_RES_KEY_SIZE 24 struct resume_key_req { char ResumeKey[COPY_CHUNK_RES_KEY_SIZE]; diff --git a/fs/cifs/smb2transport.c b/fs/cifs/smb2transport.c index d1181572758b..1ccbcf9c2c3b 100644 --- a/fs/cifs/smb2transport.c +++ b/fs/cifs/smb2transport.c @@ -734,7 +734,10 @@ smb3_crypto_aead_allocate(struct TCP_Server_Info *server) struct crypto_aead *tfm; if (!server->secmech.ccmaesencrypt) { - tfm = crypto_alloc_aead("ccm(aes)", 0, 0); + if (server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + else + tfm = crypto_alloc_aead("ccm(aes)", 0, 0); if (IS_ERR(tfm)) { cifs_dbg(VFS, "%s: Failed to alloc encrypt aead\n", __func__); @@ -744,7 +747,10 @@ smb3_crypto_aead_allocate(struct TCP_Server_Info *server) } if (!server->secmech.ccmaesdecrypt) { - tfm = crypto_alloc_aead("ccm(aes)", 0, 0); + if (server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + else + tfm = crypto_alloc_aead("ccm(aes)", 0, 0); if (IS_ERR(tfm)) { crypto_free_aead(server->secmech.ccmaesencrypt); server->secmech.ccmaesencrypt = NULL; diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index 60661b3f983a..5d6d44bfe10a 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -979,6 +979,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, }; unsigned int instance; char *buf; + struct TCP_Server_Info *server; optype = flags & CIFS_OP_MASK; @@ -990,7 +991,8 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, return -EIO; } - if (ses->server->tcpStatus == CifsExiting) + server = ses->server; + if (server->tcpStatus == CifsExiting) return -ENOENT; /* @@ -1001,7 +1003,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, * other requests. * This can be handled by the eventual session reconnect. */ - rc = wait_for_compound_request(ses->server, num_rqst, flags, + rc = wait_for_compound_request(server, num_rqst, flags, &instance); if (rc) return rc; @@ -1017,7 +1019,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, * of smb data. */ - mutex_lock(&ses->server->srv_mutex); + mutex_lock(&server->srv_mutex); /* * All the parts of the compound chain belong obtained credits from the @@ -1026,24 +1028,24 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, * we obtained credits and return -EAGAIN in such cases to let callers * handle it. */ - if (instance != ses->server->reconnect_instance) { - mutex_unlock(&ses->server->srv_mutex); + if (instance != server->reconnect_instance) { + mutex_unlock(&server->srv_mutex); for (j = 0; j < num_rqst; j++) - add_credits(ses->server, &credits[j], optype); + add_credits(server, &credits[j], optype); return -EAGAIN; } for (i = 0; i < num_rqst; i++) { - midQ[i] = ses->server->ops->setup_request(ses, &rqst[i]); + midQ[i] = server->ops->setup_request(ses, &rqst[i]); if (IS_ERR(midQ[i])) { - revert_current_mid(ses->server, i); + revert_current_mid(server, i); for (j = 0; j < i; j++) cifs_delete_mid(midQ[j]); - mutex_unlock(&ses->server->srv_mutex); + mutex_unlock(&server->srv_mutex); /* Update # of requests on wire to server */ for (j = 0; j < num_rqst; j++) - add_credits(ses->server, &credits[j], optype); + add_credits(server, &credits[j], optype); return PTR_ERR(midQ[i]); } @@ -1059,19 +1061,19 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, else midQ[i]->callback = cifs_compound_last_callback; } - cifs_in_send_inc(ses->server); - rc = smb_send_rqst(ses->server, num_rqst, rqst, flags); - cifs_in_send_dec(ses->server); + cifs_in_send_inc(server); + rc = smb_send_rqst(server, num_rqst, rqst, flags); + cifs_in_send_dec(server); for (i = 0; i < num_rqst; i++) cifs_save_when_sent(midQ[i]); if (rc < 0) { - revert_current_mid(ses->server, num_rqst); - ses->server->sequence_number -= 2; + revert_current_mid(server, num_rqst); + server->sequence_number -= 2; } - mutex_unlock(&ses->server->srv_mutex); + mutex_unlock(&server->srv_mutex); /* * If sending failed for some reason or it is an oplock break that we @@ -1079,7 +1081,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, */ if (rc < 0 || (flags & CIFS_NO_SRV_RSP)) { for (i = 0; i < num_rqst; i++) - add_credits(ses->server, &credits[i], optype); + add_credits(server, &credits[i], optype); goto out; } @@ -1099,7 +1101,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, rqst[0].rq_nvec); for (i = 0; i < num_rqst; i++) { - rc = wait_for_response(ses->server, midQ[i]); + rc = wait_for_response(server, midQ[i]); if (rc != 0) break; } @@ -1107,7 +1109,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, for (; i < num_rqst; i++) { cifs_dbg(VFS, "Cancelling wait for mid %llu cmd: %d\n", midQ[i]->mid, le16_to_cpu(midQ[i]->command)); - send_cancel(ses->server, &rqst[i], midQ[i]); + send_cancel(server, &rqst[i], midQ[i]); spin_lock(&GlobalMid_Lock); if (midQ[i]->mid_state == MID_REQUEST_SUBMITTED) { midQ[i]->mid_flags |= MID_WAIT_CANCELLED; @@ -1123,7 +1125,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, if (rc < 0) goto out; - rc = cifs_sync_mid_result(midQ[i], ses->server); + rc = cifs_sync_mid_result(midQ[i], server); if (rc != 0) { /* mark this mid as cancelled to not free it below */ cancelled_mid[i] = true; @@ -1140,14 +1142,14 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, buf = (char *)midQ[i]->resp_buf; resp_iov[i].iov_base = buf; resp_iov[i].iov_len = midQ[i]->resp_buf_size + - ses->server->vals->header_preamble_size; + server->vals->header_preamble_size; if (midQ[i]->large_buf) resp_buf_type[i] = CIFS_LARGE_BUFFER; else resp_buf_type[i] = CIFS_SMALL_BUFFER; - rc = ses->server->ops->check_receive(midQ[i], ses->server, + rc = server->ops->check_receive(midQ[i], server, flags & CIFS_LOG_ERROR); /* mark it so buf will not be freed by cifs_delete_mid */ diff --git a/fs/cifs/xattr.c b/fs/cifs/xattr.c index 50ddb795aaeb..9076150758d8 100644 --- a/fs/cifs/xattr.c +++ b/fs/cifs/xattr.c @@ -96,7 +96,6 @@ static int cifs_xattr_set(const struct xattr_handler *handler, break; case XATTR_CIFS_ACL: { -#ifdef CONFIG_CIFS_ACL struct cifs_ntsd *pacl; if (!value) @@ -117,7 +116,6 @@ static int cifs_xattr_set(const struct xattr_handler *handler, CIFS_I(inode)->time = 0; kfree(pacl); } -#endif /* CONFIG_CIFS_ACL */ break; } @@ -247,7 +245,6 @@ static int cifs_xattr_get(const struct xattr_handler *handler, break; case XATTR_CIFS_ACL: { -#ifdef CONFIG_CIFS_ACL u32 acllen; struct cifs_ntsd *pacl; @@ -270,7 +267,6 @@ static int cifs_xattr_get(const struct xattr_handler *handler, rc = acllen; kfree(pacl); } -#endif /* CONFIG_CIFS_ACL */ break; } diff --git a/fs/coda/Makefile b/fs/coda/Makefile index 1ce66819da2a..78befb8369c9 100644 --- a/fs/coda/Makefile +++ b/fs/coda/Makefile @@ -6,7 +6,8 @@ obj-$(CONFIG_CODA_FS) += coda.o coda-objs := psdev.o cache.o cnode.o inode.o dir.o file.o upcall.o \ - coda_linux.o symlink.o pioctl.o sysctl.o + coda_linux.o symlink.o pioctl.o +coda-$(CONFIG_SYSCTL) += sysctl.o # If you want debugging output, please uncomment the following line. diff --git a/fs/coda/cache.c b/fs/coda/cache.c index 201fc08a8b4f..3b8c4513118f 100644 --- a/fs/coda/cache.c +++ b/fs/coda/cache.c @@ -21,7 +21,7 @@ #include <linux/spinlock.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> +#include "coda_psdev.h" #include "coda_linux.h" #include "coda_cache.h" diff --git a/fs/coda/cnode.c b/fs/coda/cnode.c index 845b5a66952a..06855f6c7902 100644 --- a/fs/coda/cnode.c +++ b/fs/coda/cnode.c @@ -8,8 +8,8 @@ #include <linux/time.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> #include <linux/pagemap.h> +#include "coda_psdev.h" #include "coda_linux.h" static inline int coda_fideq(struct CodaFid *fid1, struct CodaFid *fid2) @@ -137,11 +137,6 @@ struct inode *coda_fid_to_inode(struct CodaFid *fid, struct super_block *sb) struct inode *inode; unsigned long hash = coda_f2i(fid); - if ( !sb ) { - pr_warn("%s: no sb!\n", __func__); - return NULL; - } - inode = ilookup5(sb, hash, coda_test_inode, fid); if ( !inode ) return NULL; @@ -153,6 +148,16 @@ struct inode *coda_fid_to_inode(struct CodaFid *fid, struct super_block *sb) return inode; } +struct coda_file_info *coda_ftoc(struct file *file) +{ + struct coda_file_info *cfi = file->private_data; + + BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + + return cfi; + +} + /* the CONTROL inode is made without asking attributes from Venus */ struct inode *coda_cnode_makectl(struct super_block *sb) { diff --git a/fs/coda/coda_fs_i.h b/fs/coda/coda_fs_i.h index d702ba1a2bf9..1763ff95d865 100644 --- a/fs/coda/coda_fs_i.h +++ b/fs/coda/coda_fs_i.h @@ -40,10 +40,9 @@ struct coda_file_info { int cfi_magic; /* magic number */ struct file *cfi_container; /* container file for this cnode */ unsigned int cfi_mapcount; /* nr of times this file is mapped */ + bool cfi_access_intent; /* is access intent supported */ }; -#define CODA_FTOC(file) ((struct coda_file_info *)((file)->private_data)) - /* flags */ #define C_VATTR 0x1 /* Validity of vattr in inode */ #define C_FLUSH 0x2 /* used after a flush */ @@ -54,6 +53,7 @@ struct inode *coda_cnode_make(struct CodaFid *, struct super_block *); struct inode *coda_iget(struct super_block *sb, struct CodaFid *fid, struct coda_vattr *attr); struct inode *coda_cnode_makectl(struct super_block *sb); struct inode *coda_fid_to_inode(struct CodaFid *fid, struct super_block *sb); +struct coda_file_info *coda_ftoc(struct file *file); void coda_replace_fid(struct inode *, struct CodaFid *, struct CodaFid *); #endif diff --git a/fs/coda/coda_int.h b/fs/coda/coda_int.h index bb0b3e0ed6c2..f82b59c9dd28 100644 --- a/fs/coda/coda_int.h +++ b/fs/coda/coda_int.h @@ -13,9 +13,19 @@ extern int coda_fake_statfs; void coda_destroy_inodecache(void); int __init coda_init_inodecache(void); int coda_fsync(struct file *coda_file, loff_t start, loff_t end, int datasync); + +#ifdef CONFIG_SYSCTL void coda_sysctl_init(void); void coda_sysctl_clean(void); +#else +static inline void coda_sysctl_init(void) +{ +} +static inline void coda_sysctl_clean(void) +{ +} +#endif #endif /* _CODA_INT_ */ diff --git a/fs/coda/coda_linux.c b/fs/coda/coda_linux.c index f3d543dd9a98..2e1a5a192074 100644 --- a/fs/coda/coda_linux.c +++ b/fs/coda/coda_linux.c @@ -18,7 +18,7 @@ #include <linux/string.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> +#include "coda_psdev.h" #include "coda_linux.h" /* initialize the debugging variables */ @@ -66,6 +66,25 @@ unsigned short coda_flags_to_cflags(unsigned short flags) return coda_flags; } +static struct timespec64 coda_to_timespec64(struct coda_timespec ts) +{ + struct timespec64 ts64 = { + .tv_sec = ts.tv_sec, + .tv_nsec = ts.tv_nsec, + }; + + return ts64; +} + +static struct coda_timespec timespec64_to_coda(struct timespec64 ts64) +{ + struct coda_timespec ts = { + .tv_sec = ts64.tv_sec, + .tv_nsec = ts64.tv_nsec, + }; + + return ts; +} /* utility functions below */ void coda_vattr_to_iattr(struct inode *inode, struct coda_vattr *attr) @@ -105,11 +124,11 @@ void coda_vattr_to_iattr(struct inode *inode, struct coda_vattr *attr) if (attr->va_size != -1) inode->i_blocks = (attr->va_size + 511) >> 9; if (attr->va_atime.tv_sec != -1) - inode->i_atime = timespec_to_timespec64(attr->va_atime); + inode->i_atime = coda_to_timespec64(attr->va_atime); if (attr->va_mtime.tv_sec != -1) - inode->i_mtime = timespec_to_timespec64(attr->va_mtime); + inode->i_mtime = coda_to_timespec64(attr->va_mtime); if (attr->va_ctime.tv_sec != -1) - inode->i_ctime = timespec_to_timespec64(attr->va_ctime); + inode->i_ctime = coda_to_timespec64(attr->va_ctime); } @@ -130,12 +149,12 @@ void coda_iattr_to_vattr(struct iattr *iattr, struct coda_vattr *vattr) vattr->va_uid = (vuid_t) -1; vattr->va_gid = (vgid_t) -1; vattr->va_size = (off_t) -1; - vattr->va_atime.tv_sec = (time_t) -1; - vattr->va_atime.tv_nsec = (time_t) -1; - vattr->va_mtime.tv_sec = (time_t) -1; - vattr->va_mtime.tv_nsec = (time_t) -1; - vattr->va_ctime.tv_sec = (time_t) -1; - vattr->va_ctime.tv_nsec = (time_t) -1; + vattr->va_atime.tv_sec = (int64_t) -1; + vattr->va_atime.tv_nsec = (long) -1; + vattr->va_mtime.tv_sec = (int64_t) -1; + vattr->va_mtime.tv_nsec = (long) -1; + vattr->va_ctime.tv_sec = (int64_t) -1; + vattr->va_ctime.tv_nsec = (long) -1; vattr->va_type = C_VNON; vattr->va_fileid = -1; vattr->va_gen = -1; @@ -175,13 +194,13 @@ void coda_iattr_to_vattr(struct iattr *iattr, struct coda_vattr *vattr) vattr->va_size = iattr->ia_size; } if ( valid & ATTR_ATIME ) { - vattr->va_atime = timespec64_to_timespec(iattr->ia_atime); + vattr->va_atime = timespec64_to_coda(iattr->ia_atime); } if ( valid & ATTR_MTIME ) { - vattr->va_mtime = timespec64_to_timespec(iattr->ia_mtime); + vattr->va_mtime = timespec64_to_coda(iattr->ia_mtime); } if ( valid & ATTR_CTIME ) { - vattr->va_ctime = timespec64_to_timespec(iattr->ia_ctime); + vattr->va_ctime = timespec64_to_coda(iattr->ia_ctime); } } diff --git a/fs/coda/coda_linux.h b/fs/coda/coda_linux.h index 126155cadfa9..d5ebd36fb2cc 100644 --- a/fs/coda/coda_linux.h +++ b/fs/coda/coda_linux.h @@ -59,22 +59,6 @@ void coda_vattr_to_iattr(struct inode *, struct coda_vattr *); void coda_iattr_to_vattr(struct iattr *, struct coda_vattr *); unsigned short coda_flags_to_cflags(unsigned short); -/* sysctl.h */ -void coda_sysctl_init(void); -void coda_sysctl_clean(void); - -#define CODA_ALLOC(ptr, cast, size) do { \ - if (size < PAGE_SIZE) \ - ptr = kzalloc((unsigned long) size, GFP_KERNEL); \ - else \ - ptr = (cast)vzalloc((unsigned long) size); \ - if (!ptr) \ - pr_warn("kernel malloc returns 0 at %s:%d\n", __FILE__, __LINE__); \ -} while (0) - - -#define CODA_FREE(ptr, size) kvfree((ptr)) - /* inode to cnode access functions */ static inline struct coda_inode_info *ITOC(struct inode *inode) diff --git a/include/linux/coda_psdev.h b/fs/coda/coda_psdev.h index 15170954aa2b..52da08c770b0 100644 --- a/include/linux/coda_psdev.h +++ b/fs/coda/coda_psdev.h @@ -3,11 +3,31 @@ #define __CODA_PSDEV_H #include <linux/backing-dev.h> +#include <linux/magic.h> #include <linux/mutex.h> -#include <uapi/linux/coda_psdev.h> + +#define CODA_PSDEV_MAJOR 67 +#define MAX_CODADEVS 5 /* how many do we allow */ struct kstatfs; +/* messages between coda filesystem in kernel and Venus */ +struct upc_req { + struct list_head uc_chain; + caddr_t uc_data; + u_short uc_flags; + u_short uc_inSize; /* Size is at most 5000 bytes */ + u_short uc_outSize; + u_short uc_opcode; /* copied from data to save lookup */ + int uc_unique; + wait_queue_head_t uc_sleep; /* process' wait queue */ +}; + +#define CODA_REQ_ASYNC 0x1 +#define CODA_REQ_READ 0x2 +#define CODA_REQ_WRITE 0x4 +#define CODA_REQ_ABORT 0x8 + /* communication pending/processing queues */ struct venus_comm { u_long vc_seq; @@ -19,7 +39,6 @@ struct venus_comm { struct mutex vc_mutex; }; - static inline struct venus_comm *coda_vcp(struct super_block *sb) { return (struct venus_comm *)((sb)->s_fs_info); @@ -30,39 +49,43 @@ int venus_rootfid(struct super_block *sb, struct CodaFid *fidp); int venus_getattr(struct super_block *sb, struct CodaFid *fid, struct coda_vattr *attr); int venus_setattr(struct super_block *, struct CodaFid *, struct coda_vattr *); -int venus_lookup(struct super_block *sb, struct CodaFid *fid, - const char *name, int length, int *type, +int venus_lookup(struct super_block *sb, struct CodaFid *fid, + const char *name, int length, int *type, struct CodaFid *resfid); int venus_close(struct super_block *sb, struct CodaFid *fid, int flags, kuid_t uid); int venus_open(struct super_block *sb, struct CodaFid *fid, int flags, struct file **f); -int venus_mkdir(struct super_block *sb, struct CodaFid *dirfid, - const char *name, int length, +int venus_mkdir(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length, struct CodaFid *newfid, struct coda_vattr *attrs); -int venus_create(struct super_block *sb, struct CodaFid *dirfid, +int venus_create(struct super_block *sb, struct CodaFid *dirfid, const char *name, int length, int excl, int mode, - struct CodaFid *newfid, struct coda_vattr *attrs) ; -int venus_rmdir(struct super_block *sb, struct CodaFid *dirfid, + struct CodaFid *newfid, struct coda_vattr *attrs); +int venus_rmdir(struct super_block *sb, struct CodaFid *dirfid, const char *name, int length); -int venus_remove(struct super_block *sb, struct CodaFid *dirfid, +int venus_remove(struct super_block *sb, struct CodaFid *dirfid, const char *name, int length); -int venus_readlink(struct super_block *sb, struct CodaFid *fid, +int venus_readlink(struct super_block *sb, struct CodaFid *fid, char *buffer, int *length); -int venus_rename(struct super_block *, struct CodaFid *new_fid, - struct CodaFid *old_fid, size_t old_length, - size_t new_length, const char *old_name, +int venus_rename(struct super_block *sb, struct CodaFid *new_fid, + struct CodaFid *old_fid, size_t old_length, + size_t new_length, const char *old_name, const char *new_name); -int venus_link(struct super_block *sb, struct CodaFid *fid, +int venus_link(struct super_block *sb, struct CodaFid *fid, struct CodaFid *dirfid, const char *name, int len ); int venus_symlink(struct super_block *sb, struct CodaFid *fid, const char *name, int len, const char *symname, int symlen); int venus_access(struct super_block *sb, struct CodaFid *fid, int mask); int venus_pioctl(struct super_block *sb, struct CodaFid *fid, unsigned int cmd, struct PioctlData *data); -int coda_downcall(struct venus_comm *vcp, int opcode, union outputArgs *out); +int coda_downcall(struct venus_comm *vcp, int opcode, union outputArgs *out, + size_t nbytes); int venus_fsync(struct super_block *sb, struct CodaFid *fid); int venus_statfs(struct dentry *dentry, struct kstatfs *sfs); +int venus_access_intent(struct super_block *sb, struct CodaFid *fid, + bool *access_intent_supported, + size_t count, loff_t ppos, int type); /* * Statistics diff --git a/fs/coda/dir.c b/fs/coda/dir.c index 00876ddadb43..ca40c2556ba6 100644 --- a/fs/coda/dir.c +++ b/fs/coda/dir.c @@ -23,7 +23,7 @@ #include <linux/uaccess.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> +#include "coda_psdev.h" #include "coda_linux.h" #include "coda_cache.h" @@ -47,8 +47,8 @@ static struct dentry *coda_lookup(struct inode *dir, struct dentry *entry, unsig int type = 0; if (length > CODA_MAXNAMLEN) { - pr_err("name too long: lookup, %s (%*s)\n", - coda_i2s(dir), (int)length, name); + pr_err("name too long: lookup, %s %zu\n", + coda_i2s(dir), length); return ERR_PTR(-ENAMETOOLONG); } @@ -356,8 +356,7 @@ static int coda_venus_readdir(struct file *coda_file, struct dir_context *ctx) ino_t ino; int ret; - cfi = CODA_FTOC(coda_file); - BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + cfi = coda_ftoc(coda_file); host_file = cfi->cfi_container; cii = ITOC(file_inode(coda_file)); @@ -426,8 +425,7 @@ static int coda_readdir(struct file *coda_file, struct dir_context *ctx) struct file *host_file; int ret; - cfi = CODA_FTOC(coda_file); - BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + cfi = coda_ftoc(coda_file); host_file = cfi->cfi_container; if (host_file->f_op->iterate || host_file->f_op->iterate_shared) { diff --git a/fs/coda/file.c b/fs/coda/file.c index 1cbc1f2298ee..128d63df5bfb 100644 --- a/fs/coda/file.c +++ b/fs/coda/file.c @@ -20,22 +20,43 @@ #include <linux/string.h> #include <linux/slab.h> #include <linux/uaccess.h> +#include <linux/uio.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> - +#include "coda_psdev.h" #include "coda_linux.h" #include "coda_int.h" +struct coda_vm_ops { + atomic_t refcnt; + struct file *coda_file; + const struct vm_operations_struct *host_vm_ops; + struct vm_operations_struct vm_ops; +}; + static ssize_t coda_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *coda_file = iocb->ki_filp; - struct coda_file_info *cfi = CODA_FTOC(coda_file); + struct inode *coda_inode = file_inode(coda_file); + struct coda_file_info *cfi = coda_ftoc(coda_file); + loff_t ki_pos = iocb->ki_pos; + size_t count = iov_iter_count(to); + ssize_t ret; - BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + ret = venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_READ); + if (ret) + goto finish_read; - return vfs_iter_read(cfi->cfi_container, to, &iocb->ki_pos, 0); + ret = vfs_iter_read(cfi->cfi_container, to, &iocb->ki_pos, 0); + +finish_read: + venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_READ_FINISH); + return ret; } static ssize_t @@ -43,13 +64,18 @@ coda_file_write_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *coda_file = iocb->ki_filp; struct inode *coda_inode = file_inode(coda_file); - struct coda_file_info *cfi = CODA_FTOC(coda_file); - struct file *host_file; + struct coda_file_info *cfi = coda_ftoc(coda_file); + struct file *host_file = cfi->cfi_container; + loff_t ki_pos = iocb->ki_pos; + size_t count = iov_iter_count(to); ssize_t ret; - BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + ret = venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_WRITE); + if (ret) + goto finish_write; - host_file = cfi->cfi_container; file_start_write(host_file); inode_lock(coda_inode); ret = vfs_iter_write(cfi->cfi_container, to, &iocb->ki_pos, 0); @@ -58,26 +84,73 @@ coda_file_write_iter(struct kiocb *iocb, struct iov_iter *to) coda_inode->i_mtime = coda_inode->i_ctime = current_time(coda_inode); inode_unlock(coda_inode); file_end_write(host_file); + +finish_write: + venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_WRITE_FINISH); return ret; } +static void +coda_vm_open(struct vm_area_struct *vma) +{ + struct coda_vm_ops *cvm_ops = + container_of(vma->vm_ops, struct coda_vm_ops, vm_ops); + + atomic_inc(&cvm_ops->refcnt); + + if (cvm_ops->host_vm_ops && cvm_ops->host_vm_ops->open) + cvm_ops->host_vm_ops->open(vma); +} + +static void +coda_vm_close(struct vm_area_struct *vma) +{ + struct coda_vm_ops *cvm_ops = + container_of(vma->vm_ops, struct coda_vm_ops, vm_ops); + + if (cvm_ops->host_vm_ops && cvm_ops->host_vm_ops->close) + cvm_ops->host_vm_ops->close(vma); + + if (atomic_dec_and_test(&cvm_ops->refcnt)) { + vma->vm_ops = cvm_ops->host_vm_ops; + fput(cvm_ops->coda_file); + kfree(cvm_ops); + } +} + static int coda_file_mmap(struct file *coda_file, struct vm_area_struct *vma) { - struct coda_file_info *cfi; + struct inode *coda_inode = file_inode(coda_file); + struct coda_file_info *cfi = coda_ftoc(coda_file); + struct file *host_file = cfi->cfi_container; + struct inode *host_inode = file_inode(host_file); struct coda_inode_info *cii; - struct file *host_file; - struct inode *coda_inode, *host_inode; - - cfi = CODA_FTOC(coda_file); - BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); - host_file = cfi->cfi_container; + struct coda_vm_ops *cvm_ops; + loff_t ppos; + size_t count; + int ret; if (!host_file->f_op->mmap) return -ENODEV; - coda_inode = file_inode(coda_file); - host_inode = file_inode(host_file); + if (WARN_ON(coda_file != vma->vm_file)) + return -EIO; + + count = vma->vm_end - vma->vm_start; + ppos = vma->vm_pgoff * PAGE_SIZE; + + ret = venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ppos, CODA_ACCESS_TYPE_MMAP); + if (ret) + return ret; + + cvm_ops = kmalloc(sizeof(struct coda_vm_ops), GFP_KERNEL); + if (!cvm_ops) + return -ENOMEM; cii = ITOC(coda_inode); spin_lock(&cii->c_lock); @@ -89,6 +162,7 @@ coda_file_mmap(struct file *coda_file, struct vm_area_struct *vma) * the container file on us! */ else if (coda_inode->i_mapping != host_inode->i_mapping) { spin_unlock(&cii->c_lock); + kfree(cvm_ops); return -EBUSY; } @@ -97,7 +171,29 @@ coda_file_mmap(struct file *coda_file, struct vm_area_struct *vma) cfi->cfi_mapcount++; spin_unlock(&cii->c_lock); - return call_mmap(host_file, vma); + vma->vm_file = get_file(host_file); + ret = call_mmap(vma->vm_file, vma); + + if (ret) { + /* if call_mmap fails, our caller will put coda_file so we + * should drop the reference to the host_file that we got. + */ + fput(host_file); + kfree(cvm_ops); + } else { + /* here we add redirects for the open/close vm_operations */ + cvm_ops->host_vm_ops = vma->vm_ops; + if (vma->vm_ops) + cvm_ops->vm_ops = *vma->vm_ops; + + cvm_ops->vm_ops.open = coda_vm_open; + cvm_ops->vm_ops.close = coda_vm_close; + cvm_ops->coda_file = coda_file; + atomic_set(&cvm_ops->refcnt, 1); + + vma->vm_ops = &cvm_ops->vm_ops; + } + return ret; } int coda_open(struct inode *coda_inode, struct file *coda_file) @@ -127,6 +223,8 @@ int coda_open(struct inode *coda_inode, struct file *coda_file) cfi->cfi_magic = CODA_MAGIC; cfi->cfi_mapcount = 0; cfi->cfi_container = host_file; + /* assume access intents are supported unless we hear otherwise */ + cfi->cfi_access_intent = true; BUG_ON(coda_file->private_data != NULL); coda_file->private_data = cfi; @@ -142,8 +240,7 @@ int coda_release(struct inode *coda_inode, struct file *coda_file) struct inode *host_inode; int err; - cfi = CODA_FTOC(coda_file); - BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + cfi = coda_ftoc(coda_file); err = venus_close(coda_inode->i_sb, coda_i2f(coda_inode), coda_flags, coda_file->f_cred->fsuid); @@ -185,8 +282,7 @@ int coda_fsync(struct file *coda_file, loff_t start, loff_t end, int datasync) return err; inode_lock(coda_inode); - cfi = CODA_FTOC(coda_file); - BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + cfi = coda_ftoc(coda_file); host_file = cfi->cfi_container; err = vfs_fsync(host_file, datasync); @@ -207,4 +303,3 @@ const struct file_operations coda_file_operations = { .fsync = coda_fsync, .splice_read = generic_file_splice_read, }; - diff --git a/fs/coda/inode.c b/fs/coda/inode.c index 23f6ebd08e80..321f56e487cb 100644 --- a/fs/coda/inode.c +++ b/fs/coda/inode.c @@ -27,7 +27,7 @@ #include <linux/vmalloc.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> +#include "coda_psdev.h" #include "coda_linux.h" #include "coda_cache.h" @@ -236,6 +236,7 @@ static void coda_put_super(struct super_block *sb) vcp->vc_sb = NULL; sb->s_fs_info = NULL; mutex_unlock(&vcp->vc_mutex); + mutex_destroy(&vcp->vc_mutex); pr_info("Bye bye.\n"); } diff --git a/fs/coda/pioctl.c b/fs/coda/pioctl.c index e0c17b7dccce..644d48c12ce8 100644 --- a/fs/coda/pioctl.c +++ b/fs/coda/pioctl.c @@ -20,8 +20,7 @@ #include <linux/uaccess.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> - +#include "coda_psdev.h" #include "coda_linux.h" /* pioctl ops */ diff --git a/fs/coda/psdev.c b/fs/coda/psdev.c index 0ceef32e6fae..240669f51eac 100644 --- a/fs/coda/psdev.c +++ b/fs/coda/psdev.c @@ -38,8 +38,7 @@ #include <linux/uaccess.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> - +#include "coda_psdev.h" #include "coda_linux.h" #include "coda_int.h" @@ -100,8 +99,12 @@ static ssize_t coda_psdev_write(struct file *file, const char __user *buf, ssize_t retval = 0, count = 0; int error; + /* make sure there is enough to copy out the (opcode, unique) values */ + if (nbytes < (2 * sizeof(u_int32_t))) + return -EINVAL; + /* Peek at the opcode, uniquefier */ - if (copy_from_user(&hdr, buf, 2 * sizeof(u_long))) + if (copy_from_user(&hdr, buf, 2 * sizeof(u_int32_t))) return -EFAULT; if (DOWNCALL(hdr.opcode)) { @@ -119,17 +122,21 @@ static ssize_t coda_psdev_write(struct file *file, const char __user *buf, hdr.opcode, hdr.unique); nbytes = size; } - CODA_ALLOC(dcbuf, union outputArgs *, nbytes); + dcbuf = kvmalloc(nbytes, GFP_KERNEL); + if (!dcbuf) { + retval = -ENOMEM; + goto out; + } if (copy_from_user(dcbuf, buf, nbytes)) { - CODA_FREE(dcbuf, nbytes); + kvfree(dcbuf); retval = -EFAULT; goto out; } /* what downcall errors does Venus handle ? */ - error = coda_downcall(vcp, hdr.opcode, dcbuf); + error = coda_downcall(vcp, hdr.opcode, dcbuf, nbytes); - CODA_FREE(dcbuf, nbytes); + kvfree(dcbuf); if (error) { pr_warn("%s: coda_downcall error: %d\n", __func__, error); @@ -182,8 +189,11 @@ static ssize_t coda_psdev_write(struct file *file, const char __user *buf, if (req->uc_opcode == CODA_OPEN_BY_FD) { struct coda_open_by_fd_out *outp = (struct coda_open_by_fd_out *)req->uc_data; - if (!outp->oh.result) + if (!outp->oh.result) { outp->fh = fget(outp->fd); + if (!outp->fh) + return -EBADF; + } } wake_up(&req->uc_sleep); @@ -252,7 +262,7 @@ static ssize_t coda_psdev_read(struct file * file, char __user * buf, goto out; } - CODA_FREE(req->uc_data, sizeof(struct coda_in_hdr)); + kvfree(req->uc_data); kfree(req); out: mutex_unlock(&vcp->vc_mutex); @@ -314,7 +324,7 @@ static int coda_psdev_release(struct inode * inode, struct file * file) /* Async requests need to be freed here */ if (req->uc_flags & CODA_REQ_ASYNC) { - CODA_FREE(req->uc_data, sizeof(struct coda_in_hdr)); + kvfree(req->uc_data); kfree(req); continue; } @@ -347,13 +357,13 @@ static const struct file_operations coda_psdev_fops = { .llseek = noop_llseek, }; -static int init_coda_psdev(void) +static int __init init_coda_psdev(void) { int i, err = 0; if (register_chrdev(CODA_PSDEV_MAJOR, "coda", &coda_psdev_fops)) { pr_err("%s: unable to get major %d\n", __func__, CODA_PSDEV_MAJOR); - return -EIO; + return -EIO; } coda_psdev_class = class_create(THIS_MODULE, "coda"); if (IS_ERR(coda_psdev_class)) { @@ -378,7 +388,7 @@ MODULE_AUTHOR("Jan Harkes, Peter J. Braam"); MODULE_DESCRIPTION("Coda Distributed File System VFS interface"); MODULE_ALIAS_CHARDEV_MAJOR(CODA_PSDEV_MAJOR); MODULE_LICENSE("GPL"); -MODULE_VERSION("6.6"); +MODULE_VERSION("7.0"); static int __init init_coda(void) { diff --git a/fs/coda/symlink.c b/fs/coda/symlink.c index 202297d156df..8907d0508198 100644 --- a/fs/coda/symlink.c +++ b/fs/coda/symlink.c @@ -17,8 +17,7 @@ #include <linux/pagemap.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> - +#include "coda_psdev.h" #include "coda_linux.h" static int coda_symlink_filler(struct file *file, struct page *page) diff --git a/fs/coda/sysctl.c b/fs/coda/sysctl.c index 0301d45000a8..fda3b702b1c5 100644 --- a/fs/coda/sysctl.c +++ b/fs/coda/sysctl.c @@ -12,7 +12,6 @@ #include "coda_int.h" -#ifdef CONFIG_SYSCTL static struct ctl_table_header *fs_table_header; static struct ctl_table coda_table[] = { @@ -62,13 +61,3 @@ void coda_sysctl_clean(void) fs_table_header = NULL; } } - -#else -void coda_sysctl_init(void) -{ -} - -void coda_sysctl_clean(void) -{ -} -#endif diff --git a/fs/coda/upcall.c b/fs/coda/upcall.c index 1175a1722411..eb3b1898da46 100644 --- a/fs/coda/upcall.c +++ b/fs/coda/upcall.c @@ -33,7 +33,7 @@ #include <linux/vfs.h> #include <linux/coda.h> -#include <linux/coda_psdev.h> +#include "coda_psdev.h" #include "coda_linux.h" #include "coda_cache.h" @@ -46,7 +46,7 @@ static void *alloc_upcall(int opcode, int size) { union inputArgs *inp; - CODA_ALLOC(inp, union inputArgs *, size); + inp = kvzalloc(size, GFP_KERNEL); if (!inp) return ERR_PTR(-ENOMEM); @@ -85,7 +85,7 @@ int venus_rootfid(struct super_block *sb, struct CodaFid *fidp) if (!error) *fidp = outp->coda_root.VFid; - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -104,7 +104,7 @@ int venus_getattr(struct super_block *sb, struct CodaFid *fid, if (!error) *attr = outp->coda_getattr.attr; - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -123,7 +123,7 @@ int venus_setattr(struct super_block *sb, struct CodaFid *fid, error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -153,7 +153,7 @@ int venus_lookup(struct super_block *sb, struct CodaFid *fid, *type = outp->coda_lookup.vtype; } - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -173,7 +173,7 @@ int venus_close(struct super_block *sb, struct CodaFid *fid, int flags, error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -194,7 +194,7 @@ int venus_open(struct super_block *sb, struct CodaFid *fid, if (!error) *fh = outp->coda_open_by_fd.fh; - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -224,7 +224,7 @@ int venus_mkdir(struct super_block *sb, struct CodaFid *dirfid, *newfid = outp->coda_mkdir.VFid; } - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -262,7 +262,7 @@ int venus_rename(struct super_block *sb, struct CodaFid *old_fid, error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -295,7 +295,7 @@ int venus_create(struct super_block *sb, struct CodaFid *dirfid, *newfid = outp->coda_create.VFid; } - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -318,7 +318,7 @@ int venus_rmdir(struct super_block *sb, struct CodaFid *dirfid, error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -340,7 +340,7 @@ int venus_remove(struct super_block *sb, struct CodaFid *dirfid, error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -370,7 +370,7 @@ int venus_readlink(struct super_block *sb, struct CodaFid *fid, *(buffer + retlen) = '\0'; } - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -398,7 +398,7 @@ int venus_link(struct super_block *sb, struct CodaFid *fid, error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -433,7 +433,7 @@ int venus_symlink(struct super_block *sb, struct CodaFid *fid, error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -449,7 +449,7 @@ int venus_fsync(struct super_block *sb, struct CodaFid *fid) inp->coda_fsync.VFid = *fid; error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -467,7 +467,7 @@ int venus_access(struct super_block *sb, struct CodaFid *fid, int mask) error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -543,7 +543,7 @@ int venus_pioctl(struct super_block *sb, struct CodaFid *fid, } exit: - CODA_FREE(inp, insize); + kvfree(inp); return error; } @@ -553,7 +553,7 @@ int venus_statfs(struct dentry *dentry, struct kstatfs *sfs) union outputArgs *outp; int insize, outsize, error; - insize = max_t(unsigned int, INSIZE(statfs), OUTSIZE(statfs)); + insize = SIZE(statfs); UPARG(CODA_STATFS); error = coda_upcall(coda_vcp(dentry->d_sb), insize, &outsize, inp); @@ -565,10 +565,51 @@ int venus_statfs(struct dentry *dentry, struct kstatfs *sfs) sfs->f_ffree = outp->coda_statfs.stat.f_ffree; } - CODA_FREE(inp, insize); + kvfree(inp); return error; } +int venus_access_intent(struct super_block *sb, struct CodaFid *fid, + bool *access_intent_supported, + size_t count, loff_t ppos, int type) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + bool finalizer = + type == CODA_ACCESS_TYPE_READ_FINISH || + type == CODA_ACCESS_TYPE_WRITE_FINISH; + + if (!*access_intent_supported && !finalizer) + return 0; + + insize = SIZE(access_intent); + UPARG(CODA_ACCESS_INTENT); + + inp->coda_access_intent.VFid = *fid; + inp->coda_access_intent.count = count; + inp->coda_access_intent.pos = ppos; + inp->coda_access_intent.type = type; + + error = coda_upcall(coda_vcp(sb), insize, + finalizer ? NULL : &outsize, inp); + + /* + * we have to free the request buffer for synchronous upcalls + * or when asynchronous upcalls fail, but not when asynchronous + * upcalls succeed + */ + if (!finalizer || error) + kvfree(inp); + + /* Chunked access is not supported or an old Coda client */ + if (error == -EOPNOTSUPP) { + *access_intent_supported = false; + error = 0; + } + return error; +} + /* * coda_upcall and coda_downcall routines. */ @@ -598,10 +639,12 @@ static void coda_unblock_signals(sigset_t *old) * has seen them, * - CODA_CLOSE or CODA_RELEASE upcall (to avoid reference count problems) * - CODA_STORE (to avoid data loss) + * - CODA_ACCESS_INTENT (to avoid reference count problems) */ #define CODA_INTERRUPTIBLE(r) (!coda_hard && \ (((r)->uc_opcode != CODA_CLOSE && \ (r)->uc_opcode != CODA_STORE && \ + (r)->uc_opcode != CODA_ACCESS_INTENT && \ (r)->uc_opcode != CODA_RELEASE) || \ (r)->uc_flags & CODA_REQ_READ)) @@ -687,21 +730,25 @@ static int coda_upcall(struct venus_comm *vcp, goto exit; } + buffer->ih.unique = ++vcp->vc_seq; + req->uc_data = (void *)buffer; - req->uc_flags = 0; + req->uc_flags = outSize ? 0 : CODA_REQ_ASYNC; req->uc_inSize = inSize; - req->uc_outSize = *outSize ? *outSize : inSize; - req->uc_opcode = ((union inputArgs *)buffer)->ih.opcode; - req->uc_unique = ++vcp->vc_seq; + req->uc_outSize = (outSize && *outSize) ? *outSize : inSize; + req->uc_opcode = buffer->ih.opcode; + req->uc_unique = buffer->ih.unique; init_waitqueue_head(&req->uc_sleep); - /* Fill in the common input args. */ - ((union inputArgs *)buffer)->ih.unique = req->uc_unique; - /* Append msg to pending queue and poke Venus. */ list_add_tail(&req->uc_chain, &vcp->vc_pending); - wake_up_interruptible(&vcp->vc_waitq); + + if (req->uc_flags & CODA_REQ_ASYNC) { + mutex_unlock(&vcp->vc_mutex); + return 0; + } + /* We can be interrupted while we wait for Venus to process * our request. If the interrupt occurs before Venus has read * the request, we dequeue and return. If it occurs after the @@ -743,20 +790,20 @@ static int coda_upcall(struct venus_comm *vcp, sig_req = kmalloc(sizeof(struct upc_req), GFP_KERNEL); if (!sig_req) goto exit; - CODA_ALLOC((sig_req->uc_data), char *, sizeof(struct coda_in_hdr)); - if (!sig_req->uc_data) { + sig_inputArgs = kvzalloc(sizeof(struct coda_in_hdr), GFP_KERNEL); + if (!sig_inputArgs) { kfree(sig_req); goto exit; } error = -EINTR; - sig_inputArgs = (union inputArgs *)sig_req->uc_data; sig_inputArgs->ih.opcode = CODA_SIGNAL; sig_inputArgs->ih.unique = req->uc_unique; sig_req->uc_flags = CODA_REQ_ASYNC; sig_req->uc_opcode = sig_inputArgs->ih.opcode; sig_req->uc_unique = sig_inputArgs->ih.unique; + sig_req->uc_data = (void *)sig_inputArgs; sig_req->uc_inSize = sizeof(struct coda_in_hdr); sig_req->uc_outSize = sizeof(struct coda_in_hdr); @@ -804,12 +851,44 @@ exit: * * CODA_REPLACE -- replace one CodaFid with another throughout the name cache */ -int coda_downcall(struct venus_comm *vcp, int opcode, union outputArgs *out) +int coda_downcall(struct venus_comm *vcp, int opcode, union outputArgs *out, + size_t nbytes) { struct inode *inode = NULL; struct CodaFid *fid = NULL, *newfid; struct super_block *sb; + /* + * Make sure we have received enough data from the cache + * manager to populate the necessary fields in the buffer + */ + switch (opcode) { + case CODA_PURGEUSER: + if (nbytes < sizeof(struct coda_purgeuser_out)) + return -EINVAL; + break; + + case CODA_ZAPDIR: + if (nbytes < sizeof(struct coda_zapdir_out)) + return -EINVAL; + break; + + case CODA_ZAPFILE: + if (nbytes < sizeof(struct coda_zapfile_out)) + return -EINVAL; + break; + + case CODA_PURGEFID: + if (nbytes < sizeof(struct coda_purgefid_out)) + return -EINVAL; + break; + + case CODA_REPLACE: + if (nbytes < sizeof(struct coda_replace_out)) + return -EINVAL; + break; + } + /* Handle invalidation requests. */ mutex_lock(&vcp->vc_mutex); sb = vcp->vc_sb; @@ -879,4 +958,3 @@ unlock_out: iput(inode); return 0; } - @@ -124,6 +124,15 @@ static int dax_is_empty_entry(void *entry) } /* + * true if the entry that was found is of a smaller order than the entry + * we were looking for + */ +static bool dax_is_conflict(void *entry) +{ + return entry == XA_RETRY_ENTRY; +} + +/* * DAX page cache entry locking */ struct exceptional_entry_key { @@ -195,11 +204,13 @@ static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all) * Look up entry in page cache, wait for it to become unlocked if it * is a DAX entry and return it. The caller must subsequently call * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry() - * if it did. + * if it did. The entry returned may have a larger order than @order. + * If @order is larger than the order of the entry found in i_pages, this + * function returns a dax_is_conflict entry. * * Must be called with the i_pages lock held. */ -static void *get_unlocked_entry(struct xa_state *xas) +static void *get_unlocked_entry(struct xa_state *xas, unsigned int order) { void *entry; struct wait_exceptional_entry_queue ewait; @@ -210,6 +221,8 @@ static void *get_unlocked_entry(struct xa_state *xas) for (;;) { entry = xas_find_conflict(xas); + if (dax_entry_order(entry) < order) + return XA_RETRY_ENTRY; if (!entry || WARN_ON_ONCE(!xa_is_value(entry)) || !dax_is_locked(entry)) return entry; @@ -254,7 +267,7 @@ static void wait_entry_unlocked(struct xa_state *xas, void *entry) static void put_unlocked_entry(struct xa_state *xas, void *entry) { /* If we were the only waiter woken, wake the next one */ - if (entry) + if (entry && dax_is_conflict(entry)) dax_wake_entry(xas, entry, false); } @@ -461,7 +474,7 @@ void dax_unlock_page(struct page *page, dax_entry_t cookie) * overlap with xarray value entries. */ static void *grab_mapping_entry(struct xa_state *xas, - struct address_space *mapping, unsigned long size_flag) + struct address_space *mapping, unsigned int order) { unsigned long index = xas->xa_index; bool pmd_downgrade = false; /* splitting PMD entry into PTE entries? */ @@ -469,20 +482,17 @@ static void *grab_mapping_entry(struct xa_state *xas, retry: xas_lock_irq(xas); - entry = get_unlocked_entry(xas); + entry = get_unlocked_entry(xas, order); if (entry) { + if (dax_is_conflict(entry)) + goto fallback; if (!xa_is_value(entry)) { xas_set_err(xas, EIO); goto out_unlock; } - if (size_flag & DAX_PMD) { - if (dax_is_pte_entry(entry)) { - put_unlocked_entry(xas, entry); - goto fallback; - } - } else { /* trying to grab a PTE entry */ + if (order == 0) { if (dax_is_pmd_entry(entry) && (dax_is_zero_entry(entry) || dax_is_empty_entry(entry))) { @@ -523,7 +533,11 @@ retry: if (entry) { dax_lock_entry(xas, entry); } else { - entry = dax_make_entry(pfn_to_pfn_t(0), size_flag | DAX_EMPTY); + unsigned long flags = DAX_EMPTY; + + if (order > 0) + flags |= DAX_PMD; + entry = dax_make_entry(pfn_to_pfn_t(0), flags); dax_lock_entry(xas, entry); if (xas_error(xas)) goto out_unlock; @@ -594,7 +608,7 @@ struct page *dax_layout_busy_page(struct address_space *mapping) if (WARN_ON_ONCE(!xa_is_value(entry))) continue; if (unlikely(dax_is_locked(entry))) - entry = get_unlocked_entry(&xas); + entry = get_unlocked_entry(&xas, 0); if (entry) page = dax_busy_page(entry); put_unlocked_entry(&xas, entry); @@ -621,7 +635,7 @@ static int __dax_invalidate_entry(struct address_space *mapping, void *entry; xas_lock_irq(&xas); - entry = get_unlocked_entry(&xas); + entry = get_unlocked_entry(&xas, 0); if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) goto out; if (!trunc && @@ -848,7 +862,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, if (unlikely(dax_is_locked(entry))) { void *old_entry = entry; - entry = get_unlocked_entry(xas); + entry = get_unlocked_entry(xas, 0); /* Entry got punched out / reallocated? */ if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) @@ -1509,7 +1523,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, * entry is already in the array, for instance), it will return * VM_FAULT_FALLBACK. */ - entry = grab_mapping_entry(&xas, mapping, DAX_PMD); + entry = grab_mapping_entry(&xas, mapping, PMD_ORDER); if (xa_is_internal(entry)) { result = xa_to_internal(entry); goto fallback; @@ -1658,11 +1672,10 @@ dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order) vm_fault_t ret; xas_lock_irq(&xas); - entry = get_unlocked_entry(&xas); + entry = get_unlocked_entry(&xas, order); /* Did we race with someone splitting entry or so? */ - if (!entry || - (order == 0 && !dax_is_pte_entry(entry)) || - (order == PMD_ORDER && !dax_is_pmd_entry(entry))) { + if (!entry || dax_is_conflict(entry) || + (order == 0 && !dax_is_pte_entry(entry))) { put_unlocked_entry(&xas, entry); xas_unlock_irq(&xas); trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 4c74c768ae43..0f9c073d78d5 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -2313,19 +2313,17 @@ SYSCALL_DEFINE6(epoll_pwait, int, epfd, struct epoll_event __user *, events, size_t, sigsetsize) { int error; - sigset_t ksigmask, sigsaved; /* * If the caller wants a certain signal mask to be set during the wait, * we apply it here. */ - error = set_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + error = set_user_sigmask(sigmask, sigsetsize); if (error) return error; error = do_epoll_wait(epfd, events, maxevents, timeout); - - restore_user_sigmask(sigmask, &sigsaved, error == -EINTR); + restore_saved_sigmask_unless(error == -EINTR); return error; } @@ -2338,19 +2336,17 @@ COMPAT_SYSCALL_DEFINE6(epoll_pwait, int, epfd, compat_size_t, sigsetsize) { long err; - sigset_t ksigmask, sigsaved; /* * If the caller wants a certain signal mask to be set during the wait, * we apply it here. */ - err = set_compat_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + err = set_compat_user_sigmask(sigmask, sigsetsize); if (err) return err; err = do_epoll_wait(epfd, events, maxevents, timeout); - - restore_user_sigmask(sigmask, &sigsaved, err == -EINTR); + restore_saved_sigmask_unless(err == -EINTR); return err; } diff --git a/fs/ext4/file.c b/fs/ext4/file.c index f4a24a46245e..70b0438dbc94 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -371,15 +371,17 @@ static const struct vm_operations_struct ext4_file_vm_ops = { static int ext4_file_mmap(struct file *file, struct vm_area_struct *vma) { struct inode *inode = file->f_mapping->host; + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct dax_device *dax_dev = sbi->s_daxdev; - if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) + if (unlikely(ext4_forced_shutdown(sbi))) return -EIO; /* - * We don't support synchronous mappings for non-DAX files. At least - * until someone comes with a sensible use case. + * We don't support synchronous mappings for non-DAX files and + * for DAX files if underneath dax_device is not synchronous. */ - if (!IS_DAX(file_inode(file)) && (vma->vm_flags & VM_SYNC)) + if (!daxdev_mapping_supported(vma, dax_dev)) return -EOPNOTSUPP; file_accessed(file); diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c index d5403b4004c9..bb0b27d88e50 100644 --- a/fs/hfsplus/xattr.c +++ b/fs/hfsplus/xattr.c @@ -407,7 +407,7 @@ static int copy_name(char *buffer, const char *xattr_name, int name_len) int offset = 0; if (!is_known_namespace(xattr_name)) { - strncpy(buffer, XATTR_MAC_OSX_PREFIX, XATTR_MAC_OSX_PREFIX_LEN); + memcpy(buffer, XATTR_MAC_OSX_PREFIX, XATTR_MAC_OSX_PREFIX_LEN); offset += XATTR_MAC_OSX_PREFIX_LEN; len += XATTR_MAC_OSX_PREFIX_LEN; } diff --git a/fs/io_uring.c b/fs/io_uring.c index d682049c07b2..e2a66e12fbc6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2400,7 +2400,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, const sigset_t __user *sig, size_t sigsz) { struct io_cq_ring *ring = ctx->cq_ring; - sigset_t ksigmask, sigsaved; int ret; if (io_cqring_events(ring) >= min_events) @@ -2410,21 +2409,17 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, #ifdef CONFIG_COMPAT if (in_compat_syscall()) ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig, - &ksigmask, &sigsaved, sigsz); + sigsz); else #endif - ret = set_user_sigmask(sig, &ksigmask, - &sigsaved, sigsz); + ret = set_user_sigmask(sig, sigsz); if (ret) return ret; } ret = wait_event_interruptible(ctx->wait, io_cqring_events(ring) >= min_events); - - if (sig) - restore_user_sigmask(sig, &sigsaved, ret == -ERESTARTSYS); - + restore_saved_sigmask_unless(ret == -ERESTARTSYS); if (ret == -ERESTARTSYS) ret = -EINTR; diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c index 679a3c8e4fb3..960f9a3c012d 100644 --- a/fs/orangefs/file.c +++ b/fs/orangefs/file.c @@ -52,7 +52,7 @@ ssize_t wait_for_direct_io(enum ORANGEFS_io_type type, struct inode *inode, struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); struct orangefs_khandle *handle = &orangefs_inode->refn.khandle; struct orangefs_kernel_op_s *new_op = NULL; - int buffer_index = -1; + int buffer_index; ssize_t ret; size_t copy_amount; @@ -134,7 +134,6 @@ populate_shared_memory: */ if (ret == -EAGAIN && op_state_purged(new_op)) { orangefs_bufmap_put(buffer_index); - buffer_index = -1; if (type == ORANGEFS_IO_WRITE) iov_iter_revert(iter, total_size); gossip_debug(GOSSIP_FILE_DEBUG, @@ -262,7 +261,6 @@ out: "%s(%pU): PUT buffer_index %d\n", __func__, handle, buffer_index); } - buffer_index = -1; } op_release(new_op); return ret; diff --git a/fs/proc/Kconfig b/fs/proc/Kconfig index 47d2651fd9dc..cb5629bd5fff 100644 --- a/fs/proc/Kconfig +++ b/fs/proc/Kconfig @@ -58,7 +58,8 @@ config PROC_VMCORE_DEVICE_DUMP snapshot. If you say Y here, the collected device dumps will be added - as ELF notes to /proc/vmcore. + as ELF notes to /proc/vmcore. You can still disable device + dump using the kernel command line option 'novmcoredd'. config PROC_SYSCTL bool "Sysctl support (/proc/sys)" if EXPERT diff --git a/fs/proc/inode.c b/fs/proc/inode.c index 5f8d215b3fd0..dbe43a50caf2 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -200,7 +200,8 @@ static loff_t proc_reg_llseek(struct file *file, loff_t offset, int whence) struct proc_dir_entry *pde = PDE(file_inode(file)); loff_t rv = -EINVAL; if (use_pde(pde)) { - loff_t (*llseek)(struct file *, loff_t, int); + typeof_member(struct file_operations, llseek) llseek; + llseek = pde->proc_fops->llseek; if (!llseek) llseek = default_llseek; @@ -212,10 +213,11 @@ static loff_t proc_reg_llseek(struct file *file, loff_t offset, int whence) static ssize_t proc_reg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { - ssize_t (*read)(struct file *, char __user *, size_t, loff_t *); struct proc_dir_entry *pde = PDE(file_inode(file)); ssize_t rv = -EIO; if (use_pde(pde)) { + typeof_member(struct file_operations, read) read; + read = pde->proc_fops->read; if (read) rv = read(file, buf, count, ppos); @@ -226,10 +228,11 @@ static ssize_t proc_reg_read(struct file *file, char __user *buf, size_t count, static ssize_t proc_reg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - ssize_t (*write)(struct file *, const char __user *, size_t, loff_t *); struct proc_dir_entry *pde = PDE(file_inode(file)); ssize_t rv = -EIO; if (use_pde(pde)) { + typeof_member(struct file_operations, write) write; + write = pde->proc_fops->write; if (write) rv = write(file, buf, count, ppos); @@ -242,8 +245,9 @@ static __poll_t proc_reg_poll(struct file *file, struct poll_table_struct *pts) { struct proc_dir_entry *pde = PDE(file_inode(file)); __poll_t rv = DEFAULT_POLLMASK; - __poll_t (*poll)(struct file *, struct poll_table_struct *); if (use_pde(pde)) { + typeof_member(struct file_operations, poll) poll; + poll = pde->proc_fops->poll; if (poll) rv = poll(file, pts); @@ -256,8 +260,9 @@ static long proc_reg_unlocked_ioctl(struct file *file, unsigned int cmd, unsigne { struct proc_dir_entry *pde = PDE(file_inode(file)); long rv = -ENOTTY; - long (*ioctl)(struct file *, unsigned int, unsigned long); if (use_pde(pde)) { + typeof_member(struct file_operations, unlocked_ioctl) ioctl; + ioctl = pde->proc_fops->unlocked_ioctl; if (ioctl) rv = ioctl(file, cmd, arg); @@ -271,8 +276,9 @@ static long proc_reg_compat_ioctl(struct file *file, unsigned int cmd, unsigned { struct proc_dir_entry *pde = PDE(file_inode(file)); long rv = -ENOTTY; - long (*compat_ioctl)(struct file *, unsigned int, unsigned long); if (use_pde(pde)) { + typeof_member(struct file_operations, compat_ioctl) compat_ioctl; + compat_ioctl = pde->proc_fops->compat_ioctl; if (compat_ioctl) rv = compat_ioctl(file, cmd, arg); @@ -286,8 +292,9 @@ static int proc_reg_mmap(struct file *file, struct vm_area_struct *vma) { struct proc_dir_entry *pde = PDE(file_inode(file)); int rv = -EIO; - int (*mmap)(struct file *, struct vm_area_struct *); if (use_pde(pde)) { + typeof_member(struct file_operations, mmap) mmap; + mmap = pde->proc_fops->mmap; if (mmap) rv = mmap(file, vma); @@ -305,7 +312,7 @@ proc_reg_get_unmapped_area(struct file *file, unsigned long orig_addr, unsigned long rv = -EIO; if (use_pde(pde)) { - typeof(proc_reg_get_unmapped_area) *get_area; + typeof_member(struct file_operations, get_unmapped_area) get_area; get_area = pde->proc_fops->get_unmapped_area; #ifdef CONFIG_MMU @@ -326,8 +333,8 @@ static int proc_reg_open(struct inode *inode, struct file *file) { struct proc_dir_entry *pde = PDE(inode); int rv = 0; - int (*open)(struct inode *, struct file *); - int (*release)(struct inode *, struct file *); + typeof_member(struct file_operations, open) open; + typeof_member(struct file_operations, release) release; struct pde_opener *pdeo; /* diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index c74570736b24..36ad1b0d6259 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -499,6 +499,10 @@ static struct inode *proc_sys_make_inode(struct super_block *sb, if (root->set_ownership) root->set_ownership(head, table, &inode->i_uid, &inode->i_gid); + else { + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + } return inode; } diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c index 57957c91c6df..7bcc92add72c 100644 --- a/fs/proc/vmcore.c +++ b/fs/proc/vmcore.c @@ -21,6 +21,7 @@ #include <linux/init.h> #include <linux/crash_dump.h> #include <linux/list.h> +#include <linux/moduleparam.h> #include <linux/mutex.h> #include <linux/vmalloc.h> #include <linux/pagemap.h> @@ -54,6 +55,9 @@ static struct proc_dir_entry *proc_vmcore; /* Device Dump list and mutex to synchronize access to list */ static LIST_HEAD(vmcoredd_list); static DEFINE_MUTEX(vmcoredd_mutex); + +static bool vmcoredd_disabled; +core_param(novmcoredd, vmcoredd_disabled, bool, 0); #endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */ /* Device Dump Size */ @@ -1452,6 +1456,11 @@ int vmcore_add_device_dump(struct vmcoredd_data *data) size_t data_size; int ret; + if (vmcoredd_disabled) { + pr_err_once("Device dump is disabled\n"); + return -EINVAL; + } + if (!data || !strlen(data->dump_name) || !data->vmcoredd_callback || !data->size) return -EINVAL; diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c index 36346dc4cec0..4517a1394c6f 100644 --- a/fs/reiserfs/journal.c +++ b/fs/reiserfs/journal.c @@ -94,7 +94,7 @@ static int journal_join(struct reiserfs_transaction_handle *th, struct super_block *sb); static void release_journal_dev(struct super_block *super, struct reiserfs_journal *journal); -static int dirty_one_transaction(struct super_block *s, +static void dirty_one_transaction(struct super_block *s, struct reiserfs_journal_list *jl); static void flush_async_commits(struct work_struct *work); static void queue_log_writer(struct super_block *s); @@ -1682,12 +1682,11 @@ next: } /* used by flush_commit_list */ -static int dirty_one_transaction(struct super_block *s, +static void dirty_one_transaction(struct super_block *s, struct reiserfs_journal_list *jl) { struct reiserfs_journal_cnode *cn; struct reiserfs_journal_list *pjl; - int ret = 0; jl->j_state |= LIST_DIRTY; cn = jl->j_realblock; @@ -1716,7 +1715,6 @@ static int dirty_one_transaction(struct super_block *s, } cn = cn->next; } - return ret; } static int kupdate_transactions(struct super_block *s, diff --git a/fs/select.c b/fs/select.c index a4d8f6e8b63c..53a0c149f528 100644 --- a/fs/select.c +++ b/fs/select.c @@ -294,12 +294,14 @@ enum poll_time_type { PT_OLD_TIMESPEC = 3, }; -static int poll_select_copy_remaining(struct timespec64 *end_time, - void __user *p, - enum poll_time_type pt_type, int ret) +static int poll_select_finish(struct timespec64 *end_time, + void __user *p, + enum poll_time_type pt_type, int ret) { struct timespec64 rts; + restore_saved_sigmask_unless(ret == -ERESTARTNOHAND); + if (!p) return ret; @@ -714,9 +716,7 @@ static int kern_select(int n, fd_set __user *inp, fd_set __user *outp, } ret = core_sys_select(n, inp, outp, exp, to); - ret = poll_select_copy_remaining(&end_time, tvp, PT_TIMEVAL, ret); - - return ret; + return poll_select_finish(&end_time, tvp, PT_TIMEVAL, ret); } SYSCALL_DEFINE5(select, int, n, fd_set __user *, inp, fd_set __user *, outp, @@ -730,7 +730,6 @@ static long do_pselect(int n, fd_set __user *inp, fd_set __user *outp, const sigset_t __user *sigmask, size_t sigsetsize, enum poll_time_type type) { - sigset_t ksigmask, sigsaved; struct timespec64 ts, end_time, *to = NULL; int ret; @@ -753,15 +752,12 @@ static long do_pselect(int n, fd_set __user *inp, fd_set __user *outp, return -EINVAL; } - ret = set_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + ret = set_user_sigmask(sigmask, sigsetsize); if (ret) return ret; ret = core_sys_select(n, inp, outp, exp, to); - restore_user_sigmask(sigmask, &sigsaved, ret == -ERESTARTNOHAND); - ret = poll_select_copy_remaining(&end_time, tsp, type, ret); - - return ret; + return poll_select_finish(&end_time, tsp, type, ret); } /* @@ -926,7 +922,7 @@ static int do_poll(struct poll_list *list, struct poll_wqueues *wait, if (!count) { count = wait->error; if (signal_pending(current)) - count = -EINTR; + count = -ERESTARTNOHAND; } if (count || timed_out) break; @@ -965,7 +961,7 @@ static int do_sys_poll(struct pollfd __user *ufds, unsigned int nfds, struct timespec64 *end_time) { struct poll_wqueues table; - int err = -EFAULT, fdcount, len, size; + int err = -EFAULT, fdcount, len; /* Allocate small arguments on the stack to save memory and be faster - use long to make sure the buffer is aligned properly on 64 bit archs to avoid unaligned access */ @@ -993,8 +989,8 @@ static int do_sys_poll(struct pollfd __user *ufds, unsigned int nfds, break; len = min(todo, POLLFD_PER_PAGE); - size = sizeof(struct poll_list) + sizeof(struct pollfd) * len; - walk = walk->next = kmalloc(size, GFP_KERNEL); + walk = walk->next = kmalloc(struct_size(walk, entries, len), + GFP_KERNEL); if (!walk) { err = -ENOMEM; goto out_fds; @@ -1041,7 +1037,7 @@ static long do_restart_poll(struct restart_block *restart_block) ret = do_sys_poll(ufds, nfds, to); - if (ret == -EINTR) { + if (ret == -ERESTARTNOHAND) { restart_block->fn = do_restart_poll; ret = -ERESTART_RESTARTBLOCK; } @@ -1062,7 +1058,7 @@ SYSCALL_DEFINE3(poll, struct pollfd __user *, ufds, unsigned int, nfds, ret = do_sys_poll(ufds, nfds, to); - if (ret == -EINTR) { + if (ret == -ERESTARTNOHAND) { struct restart_block *restart_block; restart_block = ¤t->restart_block; @@ -1086,7 +1082,6 @@ SYSCALL_DEFINE5(ppoll, struct pollfd __user *, ufds, unsigned int, nfds, struct __kernel_timespec __user *, tsp, const sigset_t __user *, sigmask, size_t, sigsetsize) { - sigset_t ksigmask, sigsaved; struct timespec64 ts, end_time, *to = NULL; int ret; @@ -1099,20 +1094,12 @@ SYSCALL_DEFINE5(ppoll, struct pollfd __user *, ufds, unsigned int, nfds, return -EINVAL; } - ret = set_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + ret = set_user_sigmask(sigmask, sigsetsize); if (ret) return ret; ret = do_sys_poll(ufds, nfds, to); - - restore_user_sigmask(sigmask, &sigsaved, ret == -EINTR); - /* We can restart this syscall, usually */ - if (ret == -EINTR) - ret = -ERESTARTNOHAND; - - ret = poll_select_copy_remaining(&end_time, tsp, PT_TIMESPEC, ret); - - return ret; + return poll_select_finish(&end_time, tsp, PT_TIMESPEC, ret); } #if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT) @@ -1121,7 +1108,6 @@ SYSCALL_DEFINE5(ppoll_time32, struct pollfd __user *, ufds, unsigned int, nfds, struct old_timespec32 __user *, tsp, const sigset_t __user *, sigmask, size_t, sigsetsize) { - sigset_t ksigmask, sigsaved; struct timespec64 ts, end_time, *to = NULL; int ret; @@ -1134,20 +1120,12 @@ SYSCALL_DEFINE5(ppoll_time32, struct pollfd __user *, ufds, unsigned int, nfds, return -EINVAL; } - ret = set_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + ret = set_user_sigmask(sigmask, sigsetsize); if (ret) return ret; ret = do_sys_poll(ufds, nfds, to); - - restore_user_sigmask(sigmask, &sigsaved, ret == -EINTR); - /* We can restart this syscall, usually */ - if (ret == -EINTR) - ret = -ERESTARTNOHAND; - - ret = poll_select_copy_remaining(&end_time, tsp, PT_OLD_TIMESPEC, ret); - - return ret; + return poll_select_finish(&end_time, tsp, PT_OLD_TIMESPEC, ret); } #endif @@ -1284,9 +1262,7 @@ static int do_compat_select(int n, compat_ulong_t __user *inp, } ret = compat_core_sys_select(n, inp, outp, exp, to); - ret = poll_select_copy_remaining(&end_time, tvp, PT_OLD_TIMEVAL, ret); - - return ret; + return poll_select_finish(&end_time, tvp, PT_OLD_TIMEVAL, ret); } COMPAT_SYSCALL_DEFINE5(select, int, n, compat_ulong_t __user *, inp, @@ -1319,7 +1295,6 @@ static long do_compat_pselect(int n, compat_ulong_t __user *inp, void __user *tsp, compat_sigset_t __user *sigmask, compat_size_t sigsetsize, enum poll_time_type type) { - sigset_t ksigmask, sigsaved; struct timespec64 ts, end_time, *to = NULL; int ret; @@ -1342,15 +1317,12 @@ static long do_compat_pselect(int n, compat_ulong_t __user *inp, return -EINVAL; } - ret = set_compat_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + ret = set_compat_user_sigmask(sigmask, sigsetsize); if (ret) return ret; ret = compat_core_sys_select(n, inp, outp, exp, to); - restore_user_sigmask(sigmask, &sigsaved, ret == -ERESTARTNOHAND); - ret = poll_select_copy_remaining(&end_time, tsp, type, ret); - - return ret; + return poll_select_finish(&end_time, tsp, type, ret); } COMPAT_SYSCALL_DEFINE6(pselect6_time64, int, n, compat_ulong_t __user *, inp, @@ -1402,7 +1374,6 @@ COMPAT_SYSCALL_DEFINE5(ppoll_time32, struct pollfd __user *, ufds, unsigned int, nfds, struct old_timespec32 __user *, tsp, const compat_sigset_t __user *, sigmask, compat_size_t, sigsetsize) { - sigset_t ksigmask, sigsaved; struct timespec64 ts, end_time, *to = NULL; int ret; @@ -1415,20 +1386,12 @@ COMPAT_SYSCALL_DEFINE5(ppoll_time32, struct pollfd __user *, ufds, return -EINVAL; } - ret = set_compat_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + ret = set_compat_user_sigmask(sigmask, sigsetsize); if (ret) return ret; ret = do_sys_poll(ufds, nfds, to); - - restore_user_sigmask(sigmask, &sigsaved, ret == -EINTR); - /* We can restart this syscall, usually */ - if (ret == -EINTR) - ret = -ERESTARTNOHAND; - - ret = poll_select_copy_remaining(&end_time, tsp, PT_OLD_TIMESPEC, ret); - - return ret; + return poll_select_finish(&end_time, tsp, PT_OLD_TIMESPEC, ret); } #endif @@ -1437,7 +1400,6 @@ COMPAT_SYSCALL_DEFINE5(ppoll_time64, struct pollfd __user *, ufds, unsigned int, nfds, struct __kernel_timespec __user *, tsp, const compat_sigset_t __user *, sigmask, compat_size_t, sigsetsize) { - sigset_t ksigmask, sigsaved; struct timespec64 ts, end_time, *to = NULL; int ret; @@ -1450,20 +1412,12 @@ COMPAT_SYSCALL_DEFINE5(ppoll_time64, struct pollfd __user *, ufds, return -EINVAL; } - ret = set_compat_user_sigmask(sigmask, &ksigmask, &sigsaved, sigsetsize); + ret = set_compat_user_sigmask(sigmask, sigsetsize); if (ret) return ret; ret = do_sys_poll(ufds, nfds, to); - - restore_user_sigmask(sigmask, &sigsaved, ret == -EINTR); - /* We can restart this syscall, usually */ - if (ret == -EINTR) - ret = -ERESTARTNOHAND; - - ret = poll_select_copy_remaining(&end_time, tsp, PT_TIMESPEC, ret); - - return ret; + return poll_select_finish(&end_time, tsp, PT_TIMESPEC, ret); } #endif diff --git a/fs/ufs/super.c b/fs/ufs/super.c index 3d247c0d92aa..4ed0dca52ec8 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -1407,11 +1407,9 @@ static int ufs_statfs(struct dentry *dentry, struct kstatfs *buf) struct super_block *sb = dentry->d_sb; struct ufs_sb_private_info *uspi= UFS_SB(sb)->s_uspi; unsigned flags = UFS_SB(sb)->s_flags; - struct ufs_super_block_third *usb3; u64 id = huge_encode_dev(sb->s_bdev->bd_dev); mutex_lock(&UFS_SB(sb)->s_lock); - usb3 = ubh_get_usb_third(uspi); if ((flags & UFS_TYPE_MASK) == UFS_TYPE_UFS2) buf->f_type = UFS2_MAGIC; diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index b74a47169297..06b68b6115bc 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -49,6 +49,7 @@ xfs-y += $(addprefix libxfs/, \ xfs_refcount_btree.o \ xfs_sb.o \ xfs_symlink_remote.o \ + xfs_trans_inode.o \ xfs_trans_resv.o \ xfs_types.o \ ) @@ -107,8 +108,7 @@ xfs-y += xfs_log.o \ xfs_rmap_item.o \ xfs_log_recover.o \ xfs_trans_ail.o \ - xfs_trans_buf.o \ - xfs_trans_inode.o + xfs_trans_buf.o # optional features xfs-$(CONFIG_XFS_QUOTA) += xfs_dquot.o \ diff --git a/fs/xfs/xfs_trans_inode.c b/fs/xfs/libxfs/xfs_trans_inode.c index 93d14e47269d..a9ad90926b87 100644 --- a/fs/xfs/xfs_trans_inode.c +++ b/fs/xfs/libxfs/xfs_trans_inode.c @@ -66,6 +66,10 @@ xfs_trans_ichgtime( inode->i_mtime = tv; if (flags & XFS_ICHGTIME_CHG) inode->i_ctime = tv; + if (flags & XFS_ICHGTIME_CREATE) { + ip->i_d.di_crtime.t_sec = (int32_t)tv.tv_sec; + ip->i_d.di_crtime.t_nsec = (int32_t)tv.tv_nsec; + } } /* diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index e93bacbd49ae..28101bbc0b78 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -1197,11 +1197,14 @@ xfs_file_mmap( struct file *filp, struct vm_area_struct *vma) { + struct dax_device *dax_dev; + + dax_dev = xfs_find_daxdev_for_inode(file_inode(filp)); /* - * We don't support synchronous mappings for non-DAX files. At least - * until someone comes with a sensible use case. + * We don't support synchronous mappings for non-DAX files and + * for DAX files if underneath dax_device is not synchronous. */ - if (!IS_DAX(file_inode(filp)) && (vma->vm_flags & VM_SYNC)) + if (!daxdev_mapping_supported(vma, dax_dev)) return -EOPNOTSUPP; file_accessed(filp); diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index 0e9bd9c83870..aa6c093d9ce9 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -104,8 +104,10 @@ extern void warn_slowpath_null(const char *file, const int line); warn_slowpath_fmt_taint(__FILE__, __LINE__, taint, arg) #else extern __printf(1, 2) void __warn_printk(const char *fmt, ...); -#define __WARN() __WARN_TAINT(TAINT_WARN) -#define __WARN_printf(arg...) do { __warn_printk(arg); __WARN(); } while (0) +#define __WARN() do { \ + printk(KERN_WARNING CUT_HERE); __WARN_TAINT(TAINT_WARN); \ +} while (0) +#define __WARN_printf(arg...) __WARN_printf_taint(TAINT_WARN, arg) #define __WARN_printf_taint(taint, arg...) \ do { __warn_printk(arg); __WARN_TAINT(taint); } while (0) #endif diff --git a/include/asm-generic/cacheflush.h b/include/asm-generic/cacheflush.h index 0dd47a6db2cf..a950a22c4890 100644 --- a/include/asm-generic/cacheflush.h +++ b/include/asm-generic/cacheflush.h @@ -5,24 +5,70 @@ /* Keep includes the same across arches. */ #include <linux/mm.h> +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 + /* * The cache doesn't need to be flushed when TLB entries change when * the cache is mapped to physical memory, not virtual memory */ -#define flush_cache_all() do { } while (0) -#define flush_cache_mm(mm) do { } while (0) -#define flush_cache_dup_mm(mm) do { } while (0) -#define flush_cache_range(vma, start, end) do { } while (0) -#define flush_cache_page(vma, vmaddr, pfn) do { } while (0) -#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 -#define flush_dcache_page(page) do { } while (0) -#define flush_dcache_mmap_lock(mapping) do { } while (0) -#define flush_dcache_mmap_unlock(mapping) do { } while (0) -#define flush_icache_range(start, end) do { } while (0) -#define flush_icache_page(vma,pg) do { } while (0) -#define flush_icache_user_range(vma,pg,adr,len) do { } while (0) -#define flush_cache_vmap(start, end) do { } while (0) -#define flush_cache_vunmap(start, end) do { } while (0) +static inline void flush_cache_all(void) +{ +} + +static inline void flush_cache_mm(struct mm_struct *mm) +{ +} + +static inline void flush_cache_dup_mm(struct mm_struct *mm) +{ +} + +static inline void flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ +} + +static inline void flush_cache_page(struct vm_area_struct *vma, + unsigned long vmaddr, + unsigned long pfn) +{ +} + +static inline void flush_dcache_page(struct page *page) +{ +} + +static inline void flush_dcache_mmap_lock(struct address_space *mapping) +{ +} + +static inline void flush_dcache_mmap_unlock(struct address_space *mapping) +{ +} + +static inline void flush_icache_range(unsigned long start, unsigned long end) +{ +} + +static inline void flush_icache_page(struct vm_area_struct *vma, + struct page *page) +{ +} + +static inline void flush_icache_user_range(struct vm_area_struct *vma, + struct page *page, + unsigned long addr, int len) +{ +} + +static inline void flush_cache_vmap(unsigned long start, unsigned long end) +{ +} + +static inline void flush_cache_vunmap(unsigned long start, unsigned long end) +{ +} #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index ca42182992a5..cd28f63bfbc7 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -246,6 +246,16 @@ #define ACPI_PROBE_TABLE(name) #endif +#ifdef CONFIG_THERMAL +#define THERMAL_TABLE(name) \ + . = ALIGN(8); \ + __##name##_thermal_table = .; \ + KEEP(*(__##name##_thermal_table)) \ + __##name##_thermal_table_end = .; +#else +#define THERMAL_TABLE(name) +#endif + #define KERNEL_DTB() \ STRUCT_ALIGN(); \ __dtb_start = .; \ @@ -615,6 +625,7 @@ IRQCHIP_OF_MATCH_TABLE() \ ACPI_PROBE_TABLE(irqchip) \ ACPI_PROBE_TABLE(timer) \ + THERMAL_TABLE(governor) \ EARLYCON_TABLE() \ LSM_TABLE() diff --git a/include/dt-bindings/clock/exynos4.h b/include/dt-bindings/clock/exynos4.h index a0439ce8e8d3..88ec3968b90a 100644 --- a/include/dt-bindings/clock/exynos4.h +++ b/include/dt-bindings/clock/exynos4.h @@ -187,6 +187,7 @@ #define CLK_MIPI_HSI 349 /* Exynos4210 only */ #define CLK_PIXELASYNCM0 351 #define CLK_PIXELASYNCM1 352 +#define CLK_ASYNC_G3D 353 /* Exynos4x12 only */ #define CLK_PWM_ISP_SCLK 379 /* Exynos4x12 only */ #define CLK_SPI0_ISP_SCLK 380 /* Exynos4x12 only */ #define CLK_SPI1_ISP_SCLK 381 /* Exynos4x12 only */ diff --git a/include/dt-bindings/clock/exynos5420.h b/include/dt-bindings/clock/exynos5420.h index 355f469943f1..02d5ac469a3d 100644 --- a/include/dt-bindings/clock/exynos5420.h +++ b/include/dt-bindings/clock/exynos5420.h @@ -60,6 +60,7 @@ #define CLK_MAU_EPLL 159 #define CLK_SCLK_HSIC_12M 160 #define CLK_SCLK_MPHY_IXTAL24 161 +#define CLK_SCLK_BPLL 162 /* gate clocks */ #define CLK_UART0 257 @@ -195,6 +196,16 @@ #define CLK_ACLK432_CAM 518 #define CLK_ACLK_FL1550_CAM 519 #define CLK_ACLK550_CAM 520 +#define CLK_CLKM_PHY0 521 +#define CLK_CLKM_PHY1 522 +#define CLK_ACLK_PPMU_DREX0_0 523 +#define CLK_ACLK_PPMU_DREX0_1 524 +#define CLK_ACLK_PPMU_DREX1_0 525 +#define CLK_ACLK_PPMU_DREX1_1 526 +#define CLK_PCLK_PPMU_DREX0_0 527 +#define CLK_PCLK_PPMU_DREX0_1 528 +#define CLK_PCLK_PPMU_DREX1_0 529 +#define CLK_PCLK_PPMU_DREX1_1 530 /* mux clocks */ #define CLK_MOUT_HDMI 640 @@ -217,6 +228,8 @@ #define CLK_MOUT_EPLL 657 #define CLK_MOUT_MAU_EPLL 658 #define CLK_MOUT_USER_MAU_EPLL 659 +#define CLK_MOUT_SCLK_SPLL 660 +#define CLK_MOUT_MX_MSPLL_CCORE_PHY 661 /* divider clocks */ #define CLK_DOUT_PIXEL 768 @@ -248,8 +261,11 @@ #define CLK_DOUT_CCLK_DREX0 794 #define CLK_DOUT_CLK2X_PHY0 795 #define CLK_DOUT_PCLK_CORE_MEM 796 +#define CLK_FF_DOUT_SPLL2 797 +#define CLK_DOUT_PCLK_DREX0 798 +#define CLK_DOUT_PCLK_DREX1 799 /* must be greater than maximal clock id */ -#define CLK_NR_CLKS 797 +#define CLK_NR_CLKS 800 #endif /* _DT_BINDINGS_CLOCK_EXYNOS_5420_H */ diff --git a/include/dt-bindings/clock/g12a-clkc.h b/include/dt-bindings/clock/g12a-clkc.h index e10470ed7c4f..b6b127e45634 100644 --- a/include/dt-bindings/clock/g12a-clkc.h +++ b/include/dt-bindings/clock/g12a-clkc.h @@ -136,5 +136,6 @@ #define CLKID_VDEC_1 204 #define CLKID_VDEC_HEVC 207 #define CLKID_VDEC_HEVCF 210 +#define CLKID_TS 212 #endif /* __G12A_CLKC_H */ diff --git a/include/dt-bindings/clock/imx8mm-clock.h b/include/dt-bindings/clock/imx8mm-clock.h index 1b4353e7b486..07e6c686f3ef 100644 --- a/include/dt-bindings/clock/imx8mm-clock.h +++ b/include/dt-bindings/clock/imx8mm-clock.h @@ -239,6 +239,15 @@ #define IMX8MM_CLK_NAND_USDHC_BUS_RAWNAND_CLK 222 -#define IMX8MM_CLK_END 223 +#define IMX8MM_CLK_GPIO1_ROOT 223 +#define IMX8MM_CLK_GPIO2_ROOT 224 +#define IMX8MM_CLK_GPIO3_ROOT 225 +#define IMX8MM_CLK_GPIO4_ROOT 226 +#define IMX8MM_CLK_GPIO5_ROOT 227 + +#define IMX8MM_CLK_SNVS_ROOT 228 +#define IMX8MM_CLK_GIC 229 + +#define IMX8MM_CLK_END 230 #endif diff --git a/include/dt-bindings/clock/imx8mq-clock.h b/include/dt-bindings/clock/imx8mq-clock.h index 6677e920dc2d..65463673d25e 100644 --- a/include/dt-bindings/clock/imx8mq-clock.h +++ b/include/dt-bindings/clock/imx8mq-clock.h @@ -400,5 +400,8 @@ #define IMX8MQ_CLK_GPIO4_ROOT 262 #define IMX8MQ_CLK_GPIO5_ROOT 263 -#define IMX8MQ_CLK_END 264 +#define IMX8MQ_CLK_SNVS_ROOT 264 +#define IMX8MQ_CLK_GIC 265 + +#define IMX8MQ_CLK_END 266 #endif /* __DT_BINDINGS_CLOCK_IMX8MQ_H */ diff --git a/include/dt-bindings/clock/meson8b-clkc.h b/include/dt-bindings/clock/meson8b-clkc.h index 47556539f0ee..68862aaf977e 100644 --- a/include/dt-bindings/clock/meson8b-clkc.h +++ b/include/dt-bindings/clock/meson8b-clkc.h @@ -112,5 +112,8 @@ #define CLKID_VDEC_HCODEC 199 #define CLKID_VDEC_2 202 #define CLKID_VDEC_HEVC 206 +#define CLKID_CTS_AMCLK 209 +#define CLKID_CTS_MCLK_I958 212 +#define CLKID_CTS_I958 213 #endif /* __MESON8B_CLKC_H */ diff --git a/include/dt-bindings/clock/mt8516-clk.h b/include/dt-bindings/clock/mt8516-clk.h index 9cfca53cd78d..816447b98edd 100644 --- a/include/dt-bindings/clock/mt8516-clk.h +++ b/include/dt-bindings/clock/mt8516-clk.h @@ -208,4 +208,21 @@ #define CLK_TOP_MSDC2_INFRA 176 #define CLK_TOP_NR_CLK 177 +/* AUDSYS */ + +#define CLK_AUD_AFE 0 +#define CLK_AUD_I2S 1 +#define CLK_AUD_22M 2 +#define CLK_AUD_24M 3 +#define CLK_AUD_INTDIR 4 +#define CLK_AUD_APLL2_TUNER 5 +#define CLK_AUD_APLL_TUNER 6 +#define CLK_AUD_HDMI 7 +#define CLK_AUD_SPDF 8 +#define CLK_AUD_ADC 9 +#define CLK_AUD_DAC 10 +#define CLK_AUD_DAC_PREDIS 11 +#define CLK_AUD_TML 12 +#define CLK_AUD_NR_CLK 13 + #endif /* _DT_BINDINGS_CLK_MT8516_H */ diff --git a/include/dt-bindings/clock/qcom,gcc-qcs404.h b/include/dt-bindings/clock/qcom,gcc-qcs404.h index 454b3f43f538..2cd62c98561f 100644 --- a/include/dt-bindings/clock/qcom,gcc-qcs404.h +++ b/include/dt-bindings/clock/qcom,gcc-qcs404.h @@ -166,5 +166,12 @@ #define GCC_PCIEPHY_0_PHY_BCR 12 #define GCC_EMAC_BCR 13 #define GCC_CDSP_RESTART 14 +#define GCC_PCIE_0_AXI_MASTER_STICKY_ARES 15 +#define GCC_PCIE_0_AHB_ARES 16 +#define GCC_PCIE_0_AXI_SLAVE_ARES 17 +#define GCC_PCIE_0_AXI_MASTER_ARES 18 +#define GCC_PCIE_0_CORE_STICKY_ARES 19 +#define GCC_PCIE_0_SLEEP_ARES 20 +#define GCC_PCIE_0_PIPE_ARES 21 #endif diff --git a/include/dt-bindings/clock/qcom,gpucc-msm8998.h b/include/dt-bindings/clock/qcom,gpucc-msm8998.h new file mode 100644 index 000000000000..2623570ee974 --- /dev/null +++ b/include/dt-bindings/clock/qcom,gpucc-msm8998.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019, Jeffrey Hugo + */ + +#ifndef _DT_BINDINGS_CLK_MSM_GPUCC_8998_H +#define _DT_BINDINGS_CLK_MSM_GPUCC_8998_H + +#define GPUPLL0 0 +#define GPUPLL0_OUT_EVEN 1 +#define RBCPR_CLK_SRC 2 +#define GFX3D_CLK_SRC 3 +#define RBBMTIMER_CLK_SRC 4 +#define GFX3D_ISENSE_CLK_SRC 5 +#define RBCPR_CLK 6 +#define GFX3D_CLK 7 +#define RBBMTIMER_CLK 8 +#define GFX3D_ISENSE_CLK 9 +#define GPUCC_CXO_CLK 10 + +#define GPU_CX_BCR 0 +#define RBCPR_BCR 1 +#define GPU_GX_BCR 2 +#define GPU_ISENSE_BCR 3 + +#define GPU_CX_GDSC 1 +#define GPU_GX_GDSC 2 + +#endif diff --git a/include/dt-bindings/clock/rk3228-cru.h b/include/dt-bindings/clock/rk3228-cru.h index 3b245e3df8da..de550ea56eeb 100644 --- a/include/dt-bindings/clock/rk3228-cru.h +++ b/include/dt-bindings/clock/rk3228-cru.h @@ -64,6 +64,7 @@ #define SCLK_WIFI 141 #define SCLK_OTGPHY0 142 #define SCLK_OTGPHY1 143 +#define SCLK_HDMI_PHY 144 /* dclk gates */ #define DCLK_VOP 190 diff --git a/include/dt-bindings/clock/rk3328-cru.h b/include/dt-bindings/clock/rk3328-cru.h index afb811340382..555b4ff660ae 100644 --- a/include/dt-bindings/clock/rk3328-cru.h +++ b/include/dt-bindings/clock/rk3328-cru.h @@ -164,6 +164,7 @@ #define PCLK_DCF 233 #define PCLK_SARADC 234 #define PCLK_ACODECPHY 235 +#define PCLK_WDT 236 /* hclk gates */ #define HCLK_PERI 308 diff --git a/include/dt-bindings/clock/stratix10-clock.h b/include/dt-bindings/clock/stratix10-clock.h index 0ac1c90a18bf..08b98e20b7cc 100644 --- a/include/dt-bindings/clock/stratix10-clock.h +++ b/include/dt-bindings/clock/stratix10-clock.h @@ -79,6 +79,8 @@ #define STRATIX10_USB_CLK 59 #define STRATIX10_SPI_M_CLK 60 #define STRATIX10_NAND_CLK 61 -#define STRATIX10_NUM_CLKS 62 +#define STRATIX10_NAND_X_CLK 62 +#define STRATIX10_NAND_ECC_CLK 63 +#define STRATIX10_NUM_CLKS 64 #endif /* __STRATIX10_CLOCK_H */ diff --git a/include/linux/acpi.h b/include/linux/acpi.h index a95cce5e82e7..9426b9aaed86 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -324,7 +324,10 @@ struct irq_domain *acpi_irq_create_hierarchy(unsigned int flags, #ifdef CONFIG_X86_IO_APIC extern int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity); #else -#define acpi_get_override_irq(gsi, trigger, polarity) (-1) +static inline int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity) +{ + return -1; +} #endif /* * This function undoes the effect of one call to acpi_register_gsi(). diff --git a/include/linux/bits.h b/include/linux/bits.h index 2b7b532c1d51..669d69441a62 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -1,13 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __LINUX_BITS_H #define __LINUX_BITS_H + +#include <linux/const.h> #include <asm/bitsperlong.h> -#define BIT(nr) (1UL << (nr)) -#define BIT_ULL(nr) (1ULL << (nr)) -#define BIT_MASK(nr) (1UL << ((nr) % BITS_PER_LONG)) +#define BIT(nr) (UL(1) << (nr)) +#define BIT_ULL(nr) (ULL(1) << (nr)) +#define BIT_MASK(nr) (UL(1) << ((nr) % BITS_PER_LONG)) #define BIT_WORD(nr) ((nr) / BITS_PER_LONG) -#define BIT_ULL_MASK(nr) (1ULL << ((nr) % BITS_PER_LONG_LONG)) +#define BIT_ULL_MASK(nr) (ULL(1) << ((nr) % BITS_PER_LONG_LONG)) #define BIT_ULL_WORD(nr) ((nr) / BITS_PER_LONG_LONG) #define BITS_PER_BYTE 8 @@ -17,10 +19,11 @@ * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000. */ #define GENMASK(h, l) \ - (((~0UL) - (1UL << (l)) + 1) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) + (((~UL(0)) - (UL(1) << (l)) + 1) & \ + (~UL(0) >> (BITS_PER_LONG - 1 - (h)))) #define GENMASK_ULL(h, l) \ - (((~0ULL) - (1ULL << (l)) + 1) & \ - (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h)))) + (((~ULL(0)) - (ULL(1) << (l)) + 1) & \ + (~ULL(0) >> (BITS_PER_LONG_LONG - 1 - (h)))) #endif /* __LINUX_BITS_H */ diff --git a/include/linux/ceph/ceph_features.h b/include/linux/ceph/ceph_features.h index 65a38c4a02a1..39e6f4c57580 100644 --- a/include/linux/ceph/ceph_features.h +++ b/include/linux/ceph/ceph_features.h @@ -211,6 +211,7 @@ DEFINE_CEPH_FEATURE_DEPRECATED(63, 1, RESERVED_BROKEN, LUMINOUS) // client-facin CEPH_FEATURE_MON_STATEFUL_SUB | \ CEPH_FEATURE_CRUSH_TUNABLES5 | \ CEPH_FEATURE_NEW_OSDOPREPLY_ENCODING | \ + CEPH_FEATURE_MSG_ADDR2 | \ CEPH_FEATURE_CEPHX_V2) #define CEPH_FEATURES_REQUIRED_DEFAULT 0 diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index 3ac0feaf2b5e..cb21c5cf12c3 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -682,7 +682,7 @@ extern const char *ceph_cap_op_name(int op); /* flags field in client cap messages (version >= 10) */ #define CEPH_CLIENT_CAPS_SYNC (1<<0) #define CEPH_CLIENT_CAPS_NO_CAPSNAP (1<<1) -#define CEPH_CLIENT_CAPS_PENDING_CAPSNAP (1<<2); +#define CEPH_CLIENT_CAPS_PENDING_CAPSNAP (1<<2) /* * caps message, used for capability callbacks, acks, requests, etc. diff --git a/include/linux/ceph/cls_lock_client.h b/include/linux/ceph/cls_lock_client.h index bea6c77d2093..17bc7584d1fe 100644 --- a/include/linux/ceph/cls_lock_client.h +++ b/include/linux/ceph/cls_lock_client.h @@ -52,4 +52,7 @@ int ceph_cls_lock_info(struct ceph_osd_client *osdc, char *lock_name, u8 *type, char **tag, struct ceph_locker **lockers, u32 *num_lockers); +int ceph_cls_assert_locked(struct ceph_osd_request *req, int which, + char *lock_name, u8 type, char *cookie, char *tag); + #endif diff --git a/include/linux/ceph/decode.h b/include/linux/ceph/decode.h index a6c2a48d42e0..450384fe487c 100644 --- a/include/linux/ceph/decode.h +++ b/include/linux/ceph/decode.h @@ -218,18 +218,27 @@ static inline void ceph_encode_timespec64(struct ceph_timespec *tv, /* * sockaddr_storage <-> ceph_sockaddr */ -static inline void ceph_encode_addr(struct ceph_entity_addr *a) +#define CEPH_ENTITY_ADDR_TYPE_NONE 0 +#define CEPH_ENTITY_ADDR_TYPE_LEGACY __cpu_to_le32(1) + +static inline void ceph_encode_banner_addr(struct ceph_entity_addr *a) { __be16 ss_family = htons(a->in_addr.ss_family); a->in_addr.ss_family = *(__u16 *)&ss_family; + + /* Banner addresses require TYPE_NONE */ + a->type = CEPH_ENTITY_ADDR_TYPE_NONE; } -static inline void ceph_decode_addr(struct ceph_entity_addr *a) +static inline void ceph_decode_banner_addr(struct ceph_entity_addr *a) { __be16 ss_family = *(__be16 *)&a->in_addr.ss_family; a->in_addr.ss_family = ntohs(ss_family); WARN_ON(a->in_addr.ss_family == 512); + a->type = CEPH_ENTITY_ADDR_TYPE_LEGACY; } +extern int ceph_decode_entity_addr(void **p, void *end, + struct ceph_entity_addr *addr); /* * encoders */ diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 337d5049ff93..82156da3c650 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -84,11 +84,13 @@ struct ceph_options { #define CEPH_MSG_MAX_MIDDLE_LEN (16*1024*1024) /* - * Handle the largest possible rbd object in one message. + * The largest possible rbd data object is 32M. + * The largest possible rbd object map object is 64M. + * * There is no limit on the size of cephfs objects, but it has to obey * rsize and wsize mount options anyway. */ -#define CEPH_MSG_MAX_DATA_LEN (32*1024*1024) +#define CEPH_MSG_MAX_DATA_LEN (64*1024*1024) #define CEPH_AUTH_NAME_DEFAULT "guest" @@ -299,10 +301,6 @@ int ceph_wait_for_latest_osdmap(struct ceph_client *client, /* pagevec.c */ extern void ceph_release_page_vector(struct page **pages, int num_pages); - -extern struct page **ceph_get_direct_page_vector(const void __user *data, - int num_pages, - bool write_page); extern void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty); extern struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags); diff --git a/include/linux/ceph/mon_client.h b/include/linux/ceph/mon_client.h index 3a4688af7455..b4d134d3312a 100644 --- a/include/linux/ceph/mon_client.h +++ b/include/linux/ceph/mon_client.h @@ -104,7 +104,6 @@ struct ceph_mon_client { #endif }; -extern struct ceph_monmap *ceph_monmap_decode(void *p, void *end); extern int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr); diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h index 2294f963dab7..ad7fe5d10dcd 100644 --- a/include/linux/ceph/osd_client.h +++ b/include/linux/ceph/osd_client.h @@ -198,9 +198,9 @@ struct ceph_osd_request { bool r_mempool; struct completion r_completion; /* private to osd_client.c */ ceph_osdc_callback_t r_callback; - struct list_head r_unsafe_item; struct inode *r_inode; /* for use by callbacks */ + struct list_head r_private_item; /* ditto */ void *r_priv; /* ditto */ /* set by submitter */ @@ -389,6 +389,14 @@ extern void ceph_osdc_handle_map(struct ceph_osd_client *osdc, void ceph_osdc_update_epoch_barrier(struct ceph_osd_client *osdc, u32 eb); void ceph_osdc_abort_requests(struct ceph_osd_client *osdc, int err); +#define osd_req_op_data(oreq, whch, typ, fld) \ +({ \ + struct ceph_osd_request *__oreq = (oreq); \ + unsigned int __whch = (whch); \ + BUG_ON(__whch >= __oreq->r_num_ops); \ + &__oreq->r_ops[__whch].typ.fld; \ +}) + extern void osd_req_op_init(struct ceph_osd_request *osd_req, unsigned int which, u16 opcode, u32 flags); @@ -497,7 +505,7 @@ int ceph_osdc_call(struct ceph_osd_client *osdc, const char *class, const char *method, unsigned int flags, struct page *req_page, size_t req_len, - struct page *resp_page, size_t *resp_len); + struct page **resp_pages, size_t *resp_len); extern int ceph_osdc_readpages(struct ceph_osd_client *osdc, struct ceph_vino vino, diff --git a/include/linux/ceph/striper.h b/include/linux/ceph/striper.h index cbd0d24b7148..3486636c0e6e 100644 --- a/include/linux/ceph/striper.h +++ b/include/linux/ceph/striper.h @@ -66,4 +66,6 @@ int ceph_extent_to_file(struct ceph_file_layout *l, struct ceph_file_extent **file_extents, u32 *num_file_extents); +u64 ceph_get_num_objects(struct ceph_file_layout *l, u64 size); + #endif diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index bb6118f79784..2ae7604783dd 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -9,8 +9,6 @@ #include <linux/of.h> #include <linux/of_clk.h> -#ifdef CONFIG_COMMON_CLK - /* * flags used across common struct clk. these flags should only affect the * top-level framework. custom flags for dealing with hardware specifics @@ -807,7 +805,14 @@ void devm_clk_hw_unregister(struct device *dev, struct clk_hw *hw); /* helper functions */ const char *__clk_get_name(const struct clk *clk); const char *clk_hw_get_name(const struct clk_hw *hw); +#ifdef CONFIG_COMMON_CLK struct clk_hw *__clk_get_hw(struct clk *clk); +#else +static inline struct clk_hw *__clk_get_hw(struct clk *clk) +{ + return (struct clk_hw *)clk; +} +#endif unsigned int clk_hw_get_num_parents(const struct clk_hw *hw); struct clk_hw *clk_hw_get_parent(const struct clk_hw *hw); struct clk_hw *clk_hw_get_parent_by_index(const struct clk_hw *hw, @@ -867,8 +872,6 @@ static inline long divider_ro_round_rate(struct clk_hw *hw, unsigned long rate, */ unsigned long clk_hw_round_rate(struct clk_hw *hw, unsigned long rate); -struct of_device_id; - struct clk_onecell_data { struct clk **clks; unsigned int clk_num; @@ -879,8 +882,6 @@ struct clk_hw_onecell_data { struct clk_hw *hws[]; }; -extern struct of_device_id __clk_of_table; - #define CLK_OF_DECLARE(name, compat, fn) OF_DECLARE_1(clk, name, compat, fn) /* @@ -904,6 +905,40 @@ extern struct of_device_id __clk_of_table; .ops = _ops, \ }) +#define CLK_HW_INIT_HW(_name, _parent, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_hws = (const struct clk_hw*[]) { _parent }, \ + .num_parents = 1, \ + .ops = _ops, \ + }) + +/* + * This macro is intended for drivers to be able to share the otherwise + * individual struct clk_hw[] compound literals created by the compiler + * when using CLK_HW_INIT_HW. It does NOT support multiple parents. + */ +#define CLK_HW_INIT_HWS(_name, _parent, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_hws = _parent, \ + .num_parents = 1, \ + .ops = _ops, \ + }) + +#define CLK_HW_INIT_FW_NAME(_name, _parent, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_data = (const struct clk_parent_data[]) { \ + { .fw_name = _parent }, \ + }, \ + .num_parents = 1, \ + .ops = _ops, \ + }) + #define CLK_HW_INIT_PARENTS(_name, _parents, _ops, _flags) \ (&(struct clk_init_data) { \ .flags = _flags, \ @@ -913,6 +948,24 @@ extern struct of_device_id __clk_of_table; .ops = _ops, \ }) +#define CLK_HW_INIT_PARENTS_HW(_name, _parents, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_hws = _parents, \ + .num_parents = ARRAY_SIZE(_parents), \ + .ops = _ops, \ + }) + +#define CLK_HW_INIT_PARENTS_DATA(_name, _parents, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_data = _parents, \ + .num_parents = ARRAY_SIZE(_parents), \ + .ops = _ops, \ + }) + #define CLK_HW_INIT_NO_PARENT(_name, _ops, _flags) \ (&(struct clk_init_data) { \ .flags = _flags, \ @@ -933,6 +986,43 @@ extern struct of_device_id __clk_of_table; _flags), \ } +#define CLK_FIXED_FACTOR_HW(_struct, _name, _parent, \ + _div, _mult, _flags) \ + struct clk_fixed_factor _struct = { \ + .div = _div, \ + .mult = _mult, \ + .hw.init = CLK_HW_INIT_HW(_name, \ + _parent, \ + &clk_fixed_factor_ops, \ + _flags), \ + } + +/* + * This macro allows the driver to reuse the _parent array for multiple + * fixed factor clk declarations. + */ +#define CLK_FIXED_FACTOR_HWS(_struct, _name, _parent, \ + _div, _mult, _flags) \ + struct clk_fixed_factor _struct = { \ + .div = _div, \ + .mult = _mult, \ + .hw.init = CLK_HW_INIT_HWS(_name, \ + _parent, \ + &clk_fixed_factor_ops, \ + _flags), \ + } + +#define CLK_FIXED_FACTOR_FW_NAME(_struct, _name, _parent, \ + _div, _mult, _flags) \ + struct clk_fixed_factor _struct = { \ + .div = _div, \ + .mult = _mult, \ + .hw.init = CLK_HW_INIT_FW_NAME(_name, \ + _parent, \ + &clk_fixed_factor_ops, \ + _flags), \ + } + #ifdef CONFIG_OF int of_clk_add_provider(struct device_node *np, struct clk *(*clk_src_get)(struct of_phandle_args *args, @@ -1019,5 +1109,4 @@ static inline int of_clk_detect_critical(struct device_node *np, int index, void clk_gate_restore_context(struct clk_hw *hw); -#endif /* CONFIG_COMMON_CLK */ #endif /* CLK_PROVIDER_H */ diff --git a/include/linux/clk.h b/include/linux/clk.h index c8e3325868bd..3c096c7a51dc 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -329,6 +329,19 @@ int __must_check clk_bulk_get(struct device *dev, int num_clks, */ int __must_check clk_bulk_get_all(struct device *dev, struct clk_bulk_data **clks); + +/** + * clk_bulk_get_optional - lookup and obtain a number of references to clock producer + * @dev: device for clock "consumer" + * @num_clks: the number of clk_bulk_data + * @clks: the clk_bulk_data table of consumer + * + * Behaves the same as clk_bulk_get() except where there is no clock producer. + * In this case, instead of returning -ENOENT, the function returns 0 and + * NULL for a clk for which a clock producer could not be determined. + */ +int __must_check clk_bulk_get_optional(struct device *dev, int num_clks, + struct clk_bulk_data *clks); /** * devm_clk_bulk_get - managed get multiple clk consumers * @dev: device for clock "consumer" @@ -344,6 +357,28 @@ int __must_check clk_bulk_get_all(struct device *dev, int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, struct clk_bulk_data *clks); /** + * devm_clk_bulk_get_optional - managed get multiple optional consumer clocks + * @dev: device for clock "consumer" + * @clks: pointer to the clk_bulk_data table of consumer + * + * Behaves the same as devm_clk_bulk_get() except where there is no clock + * producer. In this case, instead of returning -ENOENT, the function returns + * NULL for given clk. It is assumed all clocks in clk_bulk_data are optional. + * + * Returns 0 if all clocks specified in clk_bulk_data table are obtained + * successfully or for any clk there was no clk provider available, otherwise + * returns valid IS_ERR() condition containing errno. + * The implementation uses @dev and @clk_bulk_data.id to determine the + * clock consumer, and thereby the clock producer. + * The clock returned is stored in each @clk_bulk_data.clk field. + * + * Drivers must assume that the clock source is not enabled. + * + * clk_bulk_get should not be called from within interrupt context. + */ +int __must_check devm_clk_bulk_get_optional(struct device *dev, int num_clks, + struct clk_bulk_data *clks); +/** * devm_clk_bulk_get_all - managed get multiple clk consumers * @dev: device for clock "consumer" * @clks: pointer to the clk_bulk_data table of consumer @@ -715,6 +750,12 @@ static inline int __must_check clk_bulk_get(struct device *dev, int num_clks, return 0; } +static inline int __must_check clk_bulk_get_optional(struct device *dev, + int num_clks, struct clk_bulk_data *clks) +{ + return 0; +} + static inline int __must_check clk_bulk_get_all(struct device *dev, struct clk_bulk_data **clks) { @@ -738,6 +779,12 @@ static inline int __must_check devm_clk_bulk_get(struct device *dev, int num_clk return 0; } +static inline int __must_check devm_clk_bulk_get_optional(struct device *dev, + int num_clks, struct clk_bulk_data *clks) +{ + return 0; +} + static inline int __must_check devm_clk_bulk_get_all(struct device *dev, struct clk_bulk_data **clks) { diff --git a/include/linux/coda.h b/include/linux/coda.h index d30209b9cef8..0ca0c83fdb1c 100644 --- a/include/linux/coda.h +++ b/include/linux/coda.h @@ -58,8 +58,7 @@ Mellon the rights to redistribute these changes without encumbrance. #ifndef _CODA_HEADER_ #define _CODA_HEADER_ -#if defined(__linux__) typedef unsigned long long u_quad_t; -#endif + #include <uapi/linux/coda.h> #endif diff --git a/include/linux/compat.h b/include/linux/compat.h index ebddcb6cfcf8..16dafd9f4b86 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -138,8 +138,7 @@ typedef struct { compat_sigset_word sig[_COMPAT_NSIG_WORDS]; } compat_sigset_t; -int set_compat_user_sigmask(const compat_sigset_t __user *usigmask, - sigset_t *set, sigset_t *oldset, +int set_compat_user_sigmask(const compat_sigset_t __user *umask, size_t sigsetsize); struct compat_sigaction { diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 46b167fba155..536a049d7ecc 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -47,11 +47,6 @@ struct cpufreq_cpuinfo { unsigned int transition_latency; }; -struct cpufreq_user_policy { - unsigned int min; /* in kHz */ - unsigned int max; /* in kHz */ -}; - struct cpufreq_policy { /* CPUs sharing clock, require sw coordination */ cpumask_var_t cpus; /* Online CPUs only */ @@ -81,7 +76,8 @@ struct cpufreq_policy { struct work_struct update; /* if update_policy() needs to be * called, but you're in IRQ context */ - struct cpufreq_user_policy user_policy; + struct dev_pm_qos_request *min_freq_req; + struct dev_pm_qos_request *max_freq_req; struct cpufreq_frequency_table *freq_table; enum cpufreq_table_sorting freq_table_sorted; @@ -144,6 +140,9 @@ struct cpufreq_policy { /* Pointer to the cooling device if used for thermal mitigation */ struct thermal_cooling_device *cdev; + + struct notifier_block nb_min; + struct notifier_block nb_max; }; struct cpufreq_freqs { @@ -201,6 +200,7 @@ void cpufreq_cpu_release(struct cpufreq_policy *policy); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); int cpufreq_set_policy(struct cpufreq_policy *policy, struct cpufreq_policy *new_policy); +void refresh_frequency_limits(struct cpufreq_policy *policy); void cpufreq_update_policy(unsigned int cpu); void cpufreq_update_limits(unsigned int cpu); bool have_governor_per_policy(void); @@ -992,7 +992,7 @@ extern struct freq_attr *cpufreq_generic_attr[]; int cpufreq_table_validate_and_sort(struct cpufreq_policy *policy); unsigned int cpufreq_generic_get(unsigned int cpu); -int cpufreq_generic_init(struct cpufreq_policy *policy, +void cpufreq_generic_init(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table, unsigned int transition_latency); #endif /* _LINUX_CPUFREQ_H */ diff --git a/include/linux/dax.h b/include/linux/dax.h index becaea5f4488..9bd8528bd305 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -7,6 +7,9 @@ #include <linux/radix-tree.h> #include <asm/pgtable.h> +/* Flag for synchronous flush */ +#define DAXDEV_F_SYNC (1UL << 0) + typedef unsigned long dax_entry_t; struct iomap_ops; @@ -38,18 +41,40 @@ extern struct attribute_group dax_attribute_group; #if IS_ENABLED(CONFIG_DAX) struct dax_device *dax_get_by_host(const char *host); struct dax_device *alloc_dax(void *private, const char *host, - const struct dax_operations *ops); + const struct dax_operations *ops, unsigned long flags); void put_dax(struct dax_device *dax_dev); void kill_dax(struct dax_device *dax_dev); void dax_write_cache(struct dax_device *dax_dev, bool wc); bool dax_write_cache_enabled(struct dax_device *dax_dev); +bool __dax_synchronous(struct dax_device *dax_dev); +static inline bool dax_synchronous(struct dax_device *dax_dev) +{ + return __dax_synchronous(dax_dev); +} +void __set_dax_synchronous(struct dax_device *dax_dev); +static inline void set_dax_synchronous(struct dax_device *dax_dev) +{ + __set_dax_synchronous(dax_dev); +} +/* + * Check if given mapping is supported by the file / underlying device. + */ +static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, + struct dax_device *dax_dev) +{ + if (!(vma->vm_flags & VM_SYNC)) + return true; + if (!IS_DAX(file_inode(vma->vm_file))) + return false; + return dax_synchronous(dax_dev); +} #else static inline struct dax_device *dax_get_by_host(const char *host) { return NULL; } static inline struct dax_device *alloc_dax(void *private, const char *host, - const struct dax_operations *ops) + const struct dax_operations *ops, unsigned long flags) { /* * Callers should check IS_ENABLED(CONFIG_DAX) to know if this @@ -70,6 +95,18 @@ static inline bool dax_write_cache_enabled(struct dax_device *dax_dev) { return false; } +static inline bool dax_synchronous(struct dax_device *dax_dev) +{ + return true; +} +static inline void set_dax_synchronous(struct dax_device *dax_dev) +{ +} +static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, + struct dax_device *dax_dev) +{ + return !(vma->vm_flags & VM_SYNC); +} #endif struct writeback_control; diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h new file mode 100644 index 000000000000..cab6e18773da --- /dev/null +++ b/include/linux/dma/edma.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. + * Synopsys DesignWare eDMA core driver + * + * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> + */ + +#ifndef _DW_EDMA_H +#define _DW_EDMA_H + +#include <linux/device.h> +#include <linux/dmaengine.h> + +struct dw_edma; + +/** + * struct dw_edma_chip - representation of DesignWare eDMA controller hardware + * @dev: struct device of the eDMA controller + * @id: instance ID + * @irq: irq line + * @dw: struct dw_edma that is filed by dw_edma_probe() + */ +struct dw_edma_chip { + struct device *dev; + int id; + int irq; + struct dw_edma *dw; +}; + +/* Export to the platform drivers */ +#if IS_ENABLED(CONFIG_DW_EDMA) +int dw_edma_probe(struct dw_edma_chip *chip); +int dw_edma_remove(struct dw_edma_chip *chip); +#else +static inline int dw_edma_probe(struct dw_edma_chip *chip) +{ + return -ENODEV; +} + +static inline int dw_edma_remove(struct dw_edma_chip *chip) +{ + return 0; +} +#endif /* CONFIG_DW_EDMA */ + +#endif /* _DW_EDMA_H */ diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index c952f987ee57..8fcdee1c0cf9 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -1302,7 +1302,8 @@ enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie); enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx); void dma_issue_pending_all(void); struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, - dma_filter_fn fn, void *fn_param); + dma_filter_fn fn, void *fn_param, + struct device_node *np); struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name); struct dma_chan *dma_request_chan(struct device *dev, const char *name); @@ -1327,7 +1328,9 @@ static inline void dma_issue_pending_all(void) { } static inline struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, - dma_filter_fn fn, void *fn_param) + dma_filter_fn fn, + void *fn_param, + struct device_node *np) { return NULL; } @@ -1399,7 +1402,8 @@ void dma_async_device_unregister(struct dma_device *device); void dma_run_dependencies(struct dma_async_tx_descriptor *tx); struct dma_chan *dma_get_slave_channel(struct dma_chan *chan); struct dma_chan *dma_get_any_slave_channel(struct dma_device *device); -#define dma_request_channel(mask, x, y) __dma_request_channel(&(mask), x, y) +#define dma_request_channel(mask, x, y) \ + __dma_request_channel(&(mask), x, y, NULL) #define dma_request_slave_channel_compat(mask, x, y, dev, name) \ __dma_request_slave_channel_compat(&(mask), x, y, dev, name) @@ -1417,6 +1421,6 @@ static inline struct dma_chan if (!fn || !fn_param) return NULL; - return __dma_request_channel(mask, fn, fn_param); + return __dma_request_channel(mask, fn, fn_param, NULL); } #endif /* DMAENGINE_H */ diff --git a/include/linux/fpga/adi-axi-common.h b/include/linux/fpga/adi-axi-common.h new file mode 100644 index 000000000000..7fc95d5c95bb --- /dev/null +++ b/include/linux/fpga/adi-axi-common.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Analog Devices AXI common registers & definitions + * + * Copyright 2019 Analog Devices Inc. + * + * https://wiki.analog.com/resources/fpga/docs/axi_ip + * https://wiki.analog.com/resources/fpga/docs/hdl/regmap + */ + +#ifndef ADI_AXI_COMMON_H_ +#define ADI_AXI_COMMON_H_ + +#define ADI_AXI_REG_VERSION 0x0000 + +#define ADI_AXI_PCORE_VER(major, minor, patch) \ + (((major) << 16) | ((minor) << 8) | (patch)) + +#endif /* ADI_AXI_COMMON_H_ */ diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 25e2995d4a4c..8a8cb3c401b2 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -427,8 +427,8 @@ struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter); iter = ftrace_rec_iter_next(iter)) -int ftrace_update_record(struct dyn_ftrace *rec, int enable); -int ftrace_test_record(struct dyn_ftrace *rec, int enable); +int ftrace_update_record(struct dyn_ftrace *rec, bool enable); +int ftrace_test_record(struct dyn_ftrace *rec, bool enable); void ftrace_run_stop_machine(int command); unsigned long ftrace_location(unsigned long ip); unsigned long ftrace_location_range(unsigned long start, unsigned long end); diff --git a/include/linux/hwspinlock.h b/include/linux/hwspinlock.h index 0afe693be5f4..bfe7c1f1ac6d 100644 --- a/include/linux/hwspinlock.h +++ b/include/linux/hwspinlock.h @@ -14,9 +14,10 @@ #include <linux/sched.h> /* hwspinlock mode argument */ -#define HWLOCK_IRQSTATE 0x01 /* Disable interrupts, save state */ -#define HWLOCK_IRQ 0x02 /* Disable interrupts, don't save state */ -#define HWLOCK_RAW 0x03 +#define HWLOCK_IRQSTATE 0x01 /* Disable interrupts, save state */ +#define HWLOCK_IRQ 0x02 /* Disable interrupts, don't save state */ +#define HWLOCK_RAW 0x03 +#define HWLOCK_IN_ATOMIC 0x04 /* Called while in atomic context */ struct device; struct device_node; @@ -223,6 +224,23 @@ static inline int hwspin_trylock_raw(struct hwspinlock *hwlock) } /** + * hwspin_trylock_in_atomic() - attempt to lock a specific hwspinlock + * @hwlock: an hwspinlock which we want to trylock + * + * This function attempts to lock an hwspinlock, and will immediately fail + * if the hwspinlock is already taken. + * + * This function shall be called only from an atomic context. + * + * Returns 0 if we successfully locked the hwspinlock, -EBUSY if + * the hwspinlock was already taken, and -EINVAL if @hwlock is invalid. + */ +static inline int hwspin_trylock_in_atomic(struct hwspinlock *hwlock) +{ + return __hwspin_trylock(hwlock, HWLOCK_IN_ATOMIC, NULL); +} + +/** * hwspin_trylock() - attempt to lock a specific hwspinlock * @hwlock: an hwspinlock which we want to trylock * @@ -313,6 +331,28 @@ int hwspin_lock_timeout_raw(struct hwspinlock *hwlock, unsigned int to) } /** + * hwspin_lock_timeout_in_atomic() - lock an hwspinlock with timeout limit + * @hwlock: the hwspinlock to be locked + * @to: timeout value in msecs + * + * This function locks the underlying @hwlock. If the @hwlock + * is already taken, the function will busy loop waiting for it to + * be released, but give up when @timeout msecs have elapsed. + * + * This function shall be called only from an atomic context and the timeout + * value shall not exceed a few msecs. + * + * Returns 0 when the @hwlock was successfully taken, and an appropriate + * error code otherwise (most notably an -ETIMEDOUT if the @hwlock is still + * busy after @timeout msecs). The function will never sleep. + */ +static inline +int hwspin_lock_timeout_in_atomic(struct hwspinlock *hwlock, unsigned int to) +{ + return __hwspin_lock_timeout(hwlock, to, HWLOCK_IN_ATOMIC, NULL); +} + +/** * hwspin_lock_timeout() - lock an hwspinlock with timeout limit * @hwlock: the hwspinlock to be locked * @to: timeout value in msecs @@ -387,6 +427,21 @@ static inline void hwspin_unlock_raw(struct hwspinlock *hwlock) } /** + * hwspin_unlock_in_atomic() - unlock hwspinlock + * @hwlock: a previously-acquired hwspinlock which we want to unlock + * + * This function will unlock a specific hwspinlock. + * + * @hwlock must be already locked (e.g. by hwspin_trylock()) before calling + * this function: it is a bug to call unlock on a @hwlock that is already + * unlocked. + */ +static inline void hwspin_unlock_in_atomic(struct hwspinlock *hwlock) +{ + __hwspin_unlock(hwlock, HWLOCK_IN_ATOMIC, NULL); +} + +/** * hwspin_unlock() - unlock hwspinlock * @hwlock: a previously-acquired hwspinlock which we want to unlock * diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h new file mode 100644 index 000000000000..efb3ce892c20 --- /dev/null +++ b/include/linux/intel_rapl.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Data types and headers for RAPL support + * + * Copyright (C) 2019 Intel Corporation. + * + * Author: Zhang Rui <rui.zhang@intel.com> + */ + +#ifndef __INTEL_RAPL_H__ +#define __INTEL_RAPL_H__ + +#include <linux/types.h> +#include <linux/powercap.h> +#include <linux/cpuhotplug.h> + +enum rapl_domain_type { + RAPL_DOMAIN_PACKAGE, /* entire package/socket */ + RAPL_DOMAIN_PP0, /* core power plane */ + RAPL_DOMAIN_PP1, /* graphics uncore */ + RAPL_DOMAIN_DRAM, /* DRAM control_type */ + RAPL_DOMAIN_PLATFORM, /* PSys control_type */ + RAPL_DOMAIN_MAX, +}; + +enum rapl_domain_reg_id { + RAPL_DOMAIN_REG_LIMIT, + RAPL_DOMAIN_REG_STATUS, + RAPL_DOMAIN_REG_PERF, + RAPL_DOMAIN_REG_POLICY, + RAPL_DOMAIN_REG_INFO, + RAPL_DOMAIN_REG_MAX, +}; + +struct rapl_package; + +enum rapl_primitives { + ENERGY_COUNTER, + POWER_LIMIT1, + POWER_LIMIT2, + FW_LOCK, + + PL1_ENABLE, /* power limit 1, aka long term */ + PL1_CLAMP, /* allow frequency to go below OS request */ + PL2_ENABLE, /* power limit 2, aka short term, instantaneous */ + PL2_CLAMP, + + TIME_WINDOW1, /* long term */ + TIME_WINDOW2, /* short term */ + THERMAL_SPEC_POWER, + MAX_POWER, + + MIN_POWER, + MAX_TIME_WINDOW, + THROTTLED_TIME, + PRIORITY_LEVEL, + + /* below are not raw primitive data */ + AVERAGE_POWER, + NR_RAPL_PRIMITIVES, +}; + +struct rapl_domain_data { + u64 primitives[NR_RAPL_PRIMITIVES]; + unsigned long timestamp; +}; + +#define NR_POWER_LIMITS (2) +struct rapl_power_limit { + struct powercap_zone_constraint *constraint; + int prim_id; /* primitive ID used to enable */ + struct rapl_domain *domain; + const char *name; + u64 last_power_limit; +}; + +struct rapl_package; + +struct rapl_domain { + const char *name; + enum rapl_domain_type id; + u64 regs[RAPL_DOMAIN_REG_MAX]; + struct powercap_zone power_zone; + struct rapl_domain_data rdd; + struct rapl_power_limit rpl[NR_POWER_LIMITS]; + u64 attr_map; /* track capabilities */ + unsigned int state; + unsigned int domain_energy_unit; + struct rapl_package *rp; +}; + +struct reg_action { + u64 reg; + u64 mask; + u64 value; + int err; +}; + +/** + * struct rapl_if_priv: private data for different RAPL interfaces + * @control_type: Each RAPL interface must have its own powercap + * control type. + * @platform_rapl_domain: Optional. Some RAPL interface may have platform + * level RAPL control. + * @pcap_rapl_online: CPU hotplug state for each RAPL interface. + * @reg_unit: Register for getting energy/power/time unit. + * @regs: Register sets for different RAPL Domains. + * @limits: Number of power limits supported by each domain. + * @read_raw: Callback for reading RAPL interface specific + * registers. + * @write_raw: Callback for writing RAPL interface specific + * registers. + */ +struct rapl_if_priv { + struct powercap_control_type *control_type; + struct rapl_domain *platform_rapl_domain; + enum cpuhp_state pcap_rapl_online; + u64 reg_unit; + u64 regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; + int limits[RAPL_DOMAIN_MAX]; + int (*read_raw)(int cpu, struct reg_action *ra); + int (*write_raw)(int cpu, struct reg_action *ra); +}; + +/* maximum rapl package domain name: package-%d-die-%d */ +#define PACKAGE_DOMAIN_NAME_LENGTH 30 + +struct rapl_package { + unsigned int id; /* logical die id, equals physical 1-die systems */ + unsigned int nr_domains; + unsigned long domain_map; /* bit map of active domains */ + unsigned int power_unit; + unsigned int energy_unit; + unsigned int time_unit; + struct rapl_domain *domains; /* array of domains, sized at runtime */ + struct powercap_zone *power_zone; /* keep track of parent zone */ + unsigned long power_limit_irq; /* keep track of package power limit + * notify interrupt enable status. + */ + struct list_head plist; + int lead_cpu; /* one active cpu per package for access */ + /* Track active cpus */ + struct cpumask cpumask; + char name[PACKAGE_DOMAIN_NAME_LENGTH]; + struct rapl_if_priv *priv; +}; + +struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv); +struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv); +void rapl_remove_package(struct rapl_package *rp); + +int rapl_add_platform_domain(struct rapl_if_priv *priv); +void rapl_remove_platform_domain(struct rapl_if_priv *priv); + +#endif /* __INTEL_RAPL_H__ */ diff --git a/include/linux/io.h b/include/linux/io.h index 9876e5801a9d..accac822336a 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -33,6 +33,7 @@ static inline int ioremap_page_range(unsigned long addr, unsigned long end, #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP void __init ioremap_huge_init(void); +int arch_ioremap_p4d_supported(void); int arch_ioremap_pud_supported(void); int arch_ioremap_pmd_supported(void); #else diff --git a/include/linux/iversion.h b/include/linux/iversion.h index be50ef7cedab..2917ef990d43 100644 --- a/include/linux/iversion.h +++ b/include/linux/iversion.h @@ -113,6 +113,30 @@ inode_peek_iversion_raw(const struct inode *inode) } /** + * inode_set_max_iversion_raw - update i_version new value is larger + * @inode: inode to set + * @val: new i_version to set + * + * Some self-managed filesystems (e.g Ceph) will only update the i_version + * value if the new value is larger than the one we already have. + */ +static inline void +inode_set_max_iversion_raw(struct inode *inode, u64 val) +{ + u64 cur, old; + + cur = inode_peek_iversion_raw(inode); + for (;;) { + if (cur > val) + break; + old = atomic64_cmpxchg(&inode->i_version, cur, val); + if (likely(old == cur)) + break; + cur = old; + } +} + +/** * inode_set_iversion - set i_version to a particular value * @inode: inode to set * @val: new i_version value to set diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 0c9bc231107f..4fa360a13c1e 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -88,6 +88,8 @@ */ #define FIELD_SIZEOF(t, f) (sizeof(((t*)0)->f)) +#define typeof_member(T, m) typeof(((T*)0)->m) + #define DIV_ROUND_UP __KERNEL_DIV_ROUND_UP #define DIV_ROUND_DOWN_ULL(ll, d) \ diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 443d9800ca3f..04bdaf01112c 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -458,4 +458,23 @@ static inline bool is_kprobe_optinsn_slot(unsigned long addr) } #endif +/* Returns true if kprobes handled the fault */ +static nokprobe_inline bool kprobe_page_fault(struct pt_regs *regs, + unsigned int trap) +{ + if (!kprobes_built_in()) + return false; + if (user_mode(regs)) + return false; + /* + * To be potentially processing a kprobe fault and to be allowed + * to call kprobe_running(), we have to be non-preemptible. + */ + if (preemptible()) + return false; + if (!kprobe_running()) + return false; + return kprobe_fault_handler(regs, trap); +} + #endif /* _LINUX_KPROBES_H */ diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 03d5c3aece9d..7a64b3ddb408 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -11,6 +11,7 @@ #include <linux/types.h> #include <linux/uuid.h> #include <linux/spinlock.h> +#include <linux/bio.h> struct badrange_entry { u64 start; @@ -57,6 +58,9 @@ enum { */ ND_REGION_PERSIST_MEMCTRL = 2, + /* Platform provides asynchronous flush mechanism */ + ND_REGION_ASYNC = 3, + /* mark newly adjusted resources as requiring a label update */ DPA_RESOURCE_ADJUSTED = 1 << 0, }; @@ -113,6 +117,7 @@ struct nd_mapping_desc { int position; }; +struct nd_region; struct nd_region_desc { struct resource *res; struct nd_mapping_desc *mapping; @@ -125,6 +130,7 @@ struct nd_region_desc { int target_node; unsigned long flags; struct device_node *of_node; + int (*flush)(struct nd_region *nd_region, struct bio *bio); }; struct device; @@ -252,10 +258,12 @@ unsigned long nd_blk_memremap_flags(struct nd_blk_region *ndbr); unsigned int nd_region_acquire_lane(struct nd_region *nd_region); void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane); u64 nd_fletcher64(void *addr, size_t len, bool le); -void nvdimm_flush(struct nd_region *nd_region); +int nvdimm_flush(struct nd_region *nd_region, struct bio *bio); +int generic_nvdimm_flush(struct nd_region *nd_region); int nvdimm_has_flush(struct nd_region *nd_region); int nvdimm_has_cache(struct nd_region *nd_region); int nvdimm_in_overwrite(struct nvdimm *nvdimm); +bool is_nvdimm_sync(struct nd_region *nd_region); static inline int nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc) diff --git a/include/linux/lz4.h b/include/linux/lz4.h index 394e3d9213b8..b16e15b9587a 100644 --- a/include/linux/lz4.h +++ b/include/linux/lz4.h @@ -278,7 +278,7 @@ int LZ4_decompress_fast(const char *source, char *dest, int originalSize); * @compressedSize: is the precise full size of the compressed block * @maxDecompressedSize: is the size of 'dest' buffer * - * Decompresses data fom 'source' into 'dest'. + * Decompresses data from 'source' into 'dest'. * If the source stream is detected malformed, the function will * stop decoding and return a negative result. * This function is protected against buffer overflow exploits, @@ -522,7 +522,7 @@ int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, const char *dictionary, int dictSize); /** - * LZ4_decompress_fast_continue() - Decompress blocks in streaming mode + * LZ4_decompress_safe_continue() - Decompress blocks in streaming mode * @LZ4_streamDecode: the 'LZ4_streamDecode_t' structure * @source: source address of the compressed data * @dest: output buffer address of the uncompressed data @@ -530,7 +530,7 @@ int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode, * @compressedSize: is the precise full size of the compressed block * @maxDecompressedSize: is the size of 'dest' buffer * - * These decoding function allows decompression of multiple blocks + * This decoding function allows decompression of multiple blocks * in "streaming" mode. * Previously decoded blocks *must* remain available at the memory position * where they were decoded (up to 64 KB) @@ -569,7 +569,7 @@ int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode, * which must be already allocated with 'originalSize' bytes * @originalSize: is the original and therefore uncompressed size * - * These decoding function allows decompression of multiple blocks + * This decoding function allows decompression of multiple blocks * in "streaming" mode. * Previously decoded blocks *must* remain available at the memory position * where they were decoded (up to 64 KB) @@ -610,10 +610,10 @@ int LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode, * @dictStart: pointer to the start of the dictionary in memory * @dictSize: size of dictionary * - * These decoding function works the same as + * This decoding function works the same as * a combination of LZ4_setStreamDecode() followed by * LZ4_decompress_safe_continue() - * It is stand-alone, and don'tn eed a LZ4_streamDecode_t structure. + * It is stand-alone, and doesn't need an LZ4_streamDecode_t structure. * * Return: number of bytes decompressed into destination buffer * (necessarily <= maxDecompressedSize) @@ -633,10 +633,10 @@ int LZ4_decompress_safe_usingDict(const char *source, char *dest, * @dictStart: pointer to the start of the dictionary in memory * @dictSize: size of dictionary * - * These decoding function works the same as + * This decoding function works the same as * a combination of LZ4_setStreamDecode() followed by - * LZ4_decompress_safe_continue() - * It is stand-alone, and don'tn eed a LZ4_streamDecode_t structure. + * LZ4_decompress_fast_continue() + * It is stand-alone, and doesn't need an LZ4_streamDecode_t structure. * * Return: number of bytes decompressed into destination buffer * (necessarily <= maxDecompressedSize) diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index ae892eef8b82..988fde33cd7f 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -324,7 +324,7 @@ static inline void pgdat_resize_init(struct pglist_data *pgdat) {} extern bool is_mem_section_removable(unsigned long pfn, unsigned long nr_pages); extern void try_offline_node(int nid); extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); -extern void remove_memory(int nid, u64 start, u64 size); +extern int remove_memory(int nid, u64 start, u64 size); extern void __remove_memory(int nid, u64 start, u64 size); #else @@ -341,7 +341,11 @@ static inline int offline_pages(unsigned long start_pfn, unsigned long nr_pages) return -EINVAL; } -static inline void remove_memory(int nid, u64 start, u64 size) {} +static inline int remove_memory(int nid, u64 start, u64 size) +{ + return -EBUSY; +} + static inline void __remove_memory(int nid, u64 start, u64 size) {} #endif /* CONFIG_MEMORY_HOTREMOVE */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 0389c34ac529..bd6512559bed 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -547,7 +547,7 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) struct mmu_gather; struct inode; -#if !defined(__HAVE_ARCH_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) +#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) static inline int pmd_devmap(pmd_t pmd) { return 0; @@ -956,41 +956,28 @@ static inline bool put_devmap_managed_page(struct page *page) return false; } -static inline bool is_device_private_page(const struct page *page) -{ - return is_zone_device_page(page) && - page->pgmap->type == MEMORY_DEVICE_PRIVATE; -} - -#ifdef CONFIG_PCI_P2PDMA -static inline bool is_pci_p2pdma_page(const struct page *page) -{ - return is_zone_device_page(page) && - page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; -} -#else /* CONFIG_PCI_P2PDMA */ -static inline bool is_pci_p2pdma_page(const struct page *page) -{ - return false; -} -#endif /* CONFIG_PCI_P2PDMA */ - #else /* CONFIG_DEV_PAGEMAP_OPS */ static inline bool put_devmap_managed_page(struct page *page) { return false; } +#endif /* CONFIG_DEV_PAGEMAP_OPS */ static inline bool is_device_private_page(const struct page *page) { - return false; + return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) && + IS_ENABLED(CONFIG_DEVICE_PRIVATE) && + is_zone_device_page(page) && + page->pgmap->type == MEMORY_DEVICE_PRIVATE; } static inline bool is_pci_p2pdma_page(const struct page *page) { - return false; + return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) && + IS_ENABLED(CONFIG_PCI_P2PDMA) && + is_zone_device_page(page) && + page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; } -#endif /* CONFIG_DEV_PAGEMAP_OPS */ /* 127: arbitrary random number, small enough to assemble well */ #define page_ref_zero_or_close_to_overflow(page) \ @@ -1556,6 +1543,10 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, int get_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages); +int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); +int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, + struct task_struct *task, bool bypass_rlim); + /* Container for pinned pfns / pages */ struct frame_vector { unsigned int nr_allocated; /* Number of frames we have space for */ @@ -1763,7 +1754,7 @@ static inline void sync_mm_rss(struct mm_struct *mm) } #endif -#ifndef __HAVE_ARCH_PTE_DEVMAP +#ifndef CONFIG_ARCH_HAS_PTE_DEVMAP static inline int pte_devmap(pte_t pte) { return 0; @@ -2767,7 +2758,13 @@ extern int randomize_va_space; #endif const char * arch_vma_name(struct vm_area_struct *vma); +#ifdef CONFIG_MMU void print_vma_addr(char *prefix, unsigned long rip); +#else +static inline void print_vma_addr(char *prefix, unsigned long rip) +{ +} +#endif void *sparse_buffer_alloc(unsigned long size); struct page *sparse_mem_map_populate(unsigned long pnum, int nid, diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index 31013c2effd3..5229c18025e9 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -29,6 +29,11 @@ void *module_alloc(unsigned long size); /* Free memory returned from module_alloc. */ void module_memfree(void *module_region); +/* Determines if the section name is an exit section (that is only used during + * module unloading) + */ +bool module_exit_section(const char *name); + /* * Apply the given relocation to the (simplified) ELF. Return -error * or 0. diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 40015609c4b5..c842735a4f45 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2367,6 +2367,7 @@ #define PCI_DEVICE_ID_SYNOPSYS_HAPSUSB3 0xabcd #define PCI_DEVICE_ID_SYNOPSYS_HAPSUSB3_AXI 0xabce #define PCI_DEVICE_ID_SYNOPSYS_HAPSUSB31 0xabcf +#define PCI_DEVICE_ID_SYNOPSYS_EDDA 0xedda #define PCI_VENDOR_ID_USR 0x16ec diff --git a/include/linux/pfn_t.h b/include/linux/pfn_t.h index 01e8037023f7..2d9148221e9a 100644 --- a/include/linux/pfn_t.h +++ b/include/linux/pfn_t.h @@ -97,7 +97,7 @@ static inline pud_t pfn_t_pud(pfn_t pfn, pgprot_t pgprot) #endif #endif -#ifdef __HAVE_ARCH_PTE_DEVMAP +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP static inline bool pfn_t_devmap(pfn_t pfn) { const u64 flags = PFN_DEV|PFN_MAP; @@ -115,7 +115,7 @@ pmd_t pmd_mkdevmap(pmd_t pmd); defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) pud_t pud_mkdevmap(pud_t pud); #endif -#endif /* __HAVE_ARCH_PTE_DEVMAP */ +#endif /* CONFIG_ARCH_HAS_PTE_DEVMAP */ #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL static inline bool pfn_t_special(pfn_t pfn) diff --git a/include/linux/pid.h b/include/linux/pid.h index 1484db6ca8d1..2a83e434db9d 100644 --- a/include/linux/pid.h +++ b/include/linux/pid.h @@ -4,6 +4,7 @@ #include <linux/rculist.h> #include <linux/wait.h> +#include <linux/refcount.h> enum pid_type { @@ -57,7 +58,7 @@ struct upid { struct pid { - atomic_t count; + refcount_t count; unsigned int level; /* lists of tasks that use this pid */ struct hlist_head tasks[PIDTYPE_MAX]; @@ -74,7 +75,7 @@ extern const struct file_operations pidfd_fops; static inline struct pid *get_pid(struct pid *pid) { if (pid) - atomic_inc(&pid->count); + refcount_inc(&pid->count); return pid; } diff --git a/include/linux/platform_data/dma-imx.h b/include/linux/platform_data/dma-imx.h index 80f9be858bd0..281adbb26e6b 100644 --- a/include/linux/platform_data/dma-imx.h +++ b/include/linux/platform_data/dma-imx.h @@ -52,7 +52,6 @@ struct imx_dma_data { int dma_request2; /* secondary DMA request line */ enum sdma_peripheral_type peripheral_type; int priority; - struct device_node *of_node; }; static inline int imx_dma_is_ipu(struct dma_chan *chan) diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index 8551156b8dca..4802cd2c7309 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -57,7 +57,7 @@ #define ASUS_WMI_DEVID_KBD_BACKLIGHT 0x00050021 #define ASUS_WMI_DEVID_LIGHT_SENSOR 0x00050022 /* ?? */ #define ASUS_WMI_DEVID_LIGHTBAR 0x00050025 -#define ASUS_WMI_DEVID_FAN_MODE 0x00110018 +#define ASUS_WMI_DEVID_FAN_BOOST_MODE 0x00110018 /* Misc */ #define ASUS_WMI_DEVID_CAMERA 0x00060013 diff --git a/include/linux/pm_qos.h b/include/linux/pm_qos.h index 6ea1ae373d77..2aebbc5b9950 100644 --- a/include/linux/pm_qos.h +++ b/include/linux/pm_qos.h @@ -40,6 +40,8 @@ enum pm_qos_flags_status { #define PM_QOS_RESUME_LATENCY_NO_CONSTRAINT PM_QOS_LATENCY_ANY #define PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS PM_QOS_LATENCY_ANY_NS #define PM_QOS_LATENCY_TOLERANCE_DEFAULT_VALUE 0 +#define PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE 0 +#define PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE (-1) #define PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT (-1) #define PM_QOS_FLAG_NO_POWER_OFF (1 << 0) @@ -58,6 +60,8 @@ struct pm_qos_flags_request { enum dev_pm_qos_req_type { DEV_PM_QOS_RESUME_LATENCY = 1, DEV_PM_QOS_LATENCY_TOLERANCE, + DEV_PM_QOS_MIN_FREQUENCY, + DEV_PM_QOS_MAX_FREQUENCY, DEV_PM_QOS_FLAGS, }; @@ -99,10 +103,14 @@ struct pm_qos_flags { struct dev_pm_qos { struct pm_qos_constraints resume_latency; struct pm_qos_constraints latency_tolerance; + struct pm_qos_constraints min_frequency; + struct pm_qos_constraints max_frequency; struct pm_qos_flags flags; struct dev_pm_qos_request *resume_latency_req; struct dev_pm_qos_request *latency_tolerance_req; struct dev_pm_qos_request *flags_req; + struct dev_pm_qos_request *min_frequency_req; + struct dev_pm_qos_request *max_frequency_req; }; /* Action requested to pm_qos_update_target */ @@ -139,16 +147,18 @@ s32 pm_qos_read_value(struct pm_qos_constraints *c); #ifdef CONFIG_PM enum pm_qos_flags_status __dev_pm_qos_flags(struct device *dev, s32 mask); enum pm_qos_flags_status dev_pm_qos_flags(struct device *dev, s32 mask); -s32 __dev_pm_qos_read_value(struct device *dev); -s32 dev_pm_qos_read_value(struct device *dev); +s32 __dev_pm_qos_resume_latency(struct device *dev); +s32 dev_pm_qos_read_value(struct device *dev, enum dev_pm_qos_req_type type); int dev_pm_qos_add_request(struct device *dev, struct dev_pm_qos_request *req, enum dev_pm_qos_req_type type, s32 value); int dev_pm_qos_update_request(struct dev_pm_qos_request *req, s32 new_value); int dev_pm_qos_remove_request(struct dev_pm_qos_request *req); int dev_pm_qos_add_notifier(struct device *dev, - struct notifier_block *notifier); + struct notifier_block *notifier, + enum dev_pm_qos_req_type type); int dev_pm_qos_remove_notifier(struct device *dev, - struct notifier_block *notifier); + struct notifier_block *notifier, + enum dev_pm_qos_req_type type); void dev_pm_qos_constraints_init(struct device *dev); void dev_pm_qos_constraints_destroy(struct device *dev); int dev_pm_qos_add_ancestor_request(struct device *dev, @@ -174,7 +184,7 @@ static inline s32 dev_pm_qos_requested_flags(struct device *dev) return dev->power.qos->flags_req->data.flr.flags; } -static inline s32 dev_pm_qos_raw_read_value(struct device *dev) +static inline s32 dev_pm_qos_raw_resume_latency(struct device *dev) { return IS_ERR_OR_NULL(dev->power.qos) ? PM_QOS_RESUME_LATENCY_NO_CONSTRAINT : @@ -187,10 +197,24 @@ static inline enum pm_qos_flags_status __dev_pm_qos_flags(struct device *dev, static inline enum pm_qos_flags_status dev_pm_qos_flags(struct device *dev, s32 mask) { return PM_QOS_FLAGS_UNDEFINED; } -static inline s32 __dev_pm_qos_read_value(struct device *dev) - { return PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; } -static inline s32 dev_pm_qos_read_value(struct device *dev) +static inline s32 __dev_pm_qos_resume_latency(struct device *dev) { return PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; } +static inline s32 dev_pm_qos_read_value(struct device *dev, + enum dev_pm_qos_req_type type) +{ + switch (type) { + case DEV_PM_QOS_RESUME_LATENCY: + return PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; + case DEV_PM_QOS_MIN_FREQUENCY: + return PM_QOS_MIN_FREQUENCY_DEFAULT_VALUE; + case DEV_PM_QOS_MAX_FREQUENCY: + return PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE; + default: + WARN_ON(1); + return 0; + } +} + static inline int dev_pm_qos_add_request(struct device *dev, struct dev_pm_qos_request *req, enum dev_pm_qos_req_type type, @@ -202,10 +226,12 @@ static inline int dev_pm_qos_update_request(struct dev_pm_qos_request *req, static inline int dev_pm_qos_remove_request(struct dev_pm_qos_request *req) { return 0; } static inline int dev_pm_qos_add_notifier(struct device *dev, - struct notifier_block *notifier) + struct notifier_block *notifier, + enum dev_pm_qos_req_type type) { return 0; } static inline int dev_pm_qos_remove_notifier(struct device *dev, - struct notifier_block *notifier) + struct notifier_block *notifier, + enum dev_pm_qos_req_type type) { return 0; } static inline void dev_pm_qos_constraints_init(struct device *dev) { @@ -241,7 +267,7 @@ static inline s32 dev_pm_qos_requested_resume_latency(struct device *dev) return PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; } static inline s32 dev_pm_qos_requested_flags(struct device *dev) { return 0; } -static inline s32 dev_pm_qos_raw_read_value(struct device *dev) +static inline s32 dev_pm_qos_raw_resume_latency(struct device *dev) { return PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; } diff --git a/include/linux/poison.h b/include/linux/poison.h index d6d980a681c7..df34330b4e34 100644 --- a/include/linux/poison.h +++ b/include/linux/poison.h @@ -21,7 +21,7 @@ * non-initialized list entries. */ #define LIST_POISON1 ((void *) 0x100 + POISON_POINTER_DELTA) -#define LIST_POISON2 ((void *) 0x200 + POISON_POINTER_DELTA) +#define LIST_POISON2 ((void *) 0x122 + POISON_POINTER_DELTA) /********** include/linux/timer.h **********/ /* diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h index e6337fce08f2..1fd61a9af45c 100644 --- a/include/linux/rbtree.h +++ b/include/linux/rbtree.h @@ -32,25 +32,9 @@ struct rb_root { struct rb_node *rb_node; }; -/* - * Leftmost-cached rbtrees. - * - * We do not cache the rightmost node based on footprint - * size vs number of potential users that could benefit - * from O(1) rb_last(). Just not worth it, users that want - * this feature can always implement the logic explicitly. - * Furthermore, users that want to cache both pointers may - * find it a bit asymmetric, but that's ok. - */ -struct rb_root_cached { - struct rb_root rb_root; - struct rb_node *rb_leftmost; -}; - #define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3)) #define RB_ROOT (struct rb_root) { NULL, } -#define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL } #define rb_entry(ptr, type, member) container_of(ptr, type, member) #define RB_EMPTY_ROOT(root) (READ_ONCE((root)->rb_node) == NULL) @@ -72,12 +56,6 @@ extern struct rb_node *rb_prev(const struct rb_node *); extern struct rb_node *rb_first(const struct rb_root *); extern struct rb_node *rb_last(const struct rb_root *); -extern void rb_insert_color_cached(struct rb_node *, - struct rb_root_cached *, bool); -extern void rb_erase_cached(struct rb_node *node, struct rb_root_cached *); -/* Same as rb_first(), but O(1) */ -#define rb_first_cached(root) (root)->rb_leftmost - /* Postorder iteration - always visit the parent after its children */ extern struct rb_node *rb_first_postorder(const struct rb_root *); extern struct rb_node *rb_next_postorder(const struct rb_node *); @@ -87,8 +65,6 @@ extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, struct rb_root *root); extern void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new, struct rb_root *root); -extern void rb_replace_node_cached(struct rb_node *victim, struct rb_node *new, - struct rb_root_cached *root); static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, struct rb_node **rb_link) @@ -136,4 +112,50 @@ static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent typeof(*pos), field); 1; }); \ pos = n) +/* + * Leftmost-cached rbtrees. + * + * We do not cache the rightmost node based on footprint + * size vs number of potential users that could benefit + * from O(1) rb_last(). Just not worth it, users that want + * this feature can always implement the logic explicitly. + * Furthermore, users that want to cache both pointers may + * find it a bit asymmetric, but that's ok. + */ +struct rb_root_cached { + struct rb_root rb_root; + struct rb_node *rb_leftmost; +}; + +#define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL } + +/* Same as rb_first(), but O(1) */ +#define rb_first_cached(root) (root)->rb_leftmost + +static inline void rb_insert_color_cached(struct rb_node *node, + struct rb_root_cached *root, + bool leftmost) +{ + if (leftmost) + root->rb_leftmost = node; + rb_insert_color(node, &root->rb_root); +} + +static inline void rb_erase_cached(struct rb_node *node, + struct rb_root_cached *root) +{ + if (root->rb_leftmost == node) + root->rb_leftmost = rb_next(node); + rb_erase(node, &root->rb_root); +} + +static inline void rb_replace_node_cached(struct rb_node *victim, + struct rb_node *new, + struct rb_root_cached *root) +{ + if (root->rb_leftmost == victim) + root->rb_leftmost = new; + rb_replace_node(victim, new, &root->rb_root); +} + #endif /* _LINUX_RBTREE_H */ diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index 0f902ccb48b0..179faab29f52 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -30,10 +30,9 @@ struct rb_augment_callbacks { void (*rotate)(struct rb_node *old, struct rb_node *new); }; -extern void __rb_insert_augmented(struct rb_node *node, - struct rb_root *root, - bool newleft, struct rb_node **leftmost, +extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); + /* * Fixup the rbtree and update the augmented information when rebalancing. * @@ -48,7 +47,7 @@ static inline void rb_insert_augmented(struct rb_node *node, struct rb_root *root, const struct rb_augment_callbacks *augment) { - __rb_insert_augmented(node, root, false, NULL, augment->rotate); + __rb_insert_augmented(node, root, augment->rotate); } static inline void @@ -56,8 +55,9 @@ rb_insert_augmented_cached(struct rb_node *node, struct rb_root_cached *root, bool newleft, const struct rb_augment_callbacks *augment) { - __rb_insert_augmented(node, &root->rb_root, - newleft, &root->rb_leftmost, augment->rotate); + if (newleft) + root->rb_leftmost = node; + rb_insert_augmented(node, &root->rb_root, augment); } #define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \ @@ -150,7 +150,6 @@ extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root, static __always_inline struct rb_node * __rb_erase_augmented(struct rb_node *node, struct rb_root *root, - struct rb_node **leftmost, const struct rb_augment_callbacks *augment) { struct rb_node *child = node->rb_right; @@ -158,9 +157,6 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root, struct rb_node *parent, *rebalance; unsigned long pc; - if (leftmost && node == *leftmost) - *leftmost = rb_next(node); - if (!tmp) { /* * Case 1: node to erase has no more than 1 child (easy!) @@ -260,8 +256,7 @@ static __always_inline void rb_erase_augmented(struct rb_node *node, struct rb_root *root, const struct rb_augment_callbacks *augment) { - struct rb_node *rebalance = __rb_erase_augmented(node, root, - NULL, augment); + struct rb_node *rebalance = __rb_erase_augmented(node, root, augment); if (rebalance) __rb_erase_color(rebalance, root, augment->rotate); } @@ -270,11 +265,9 @@ static __always_inline void rb_erase_augmented_cached(struct rb_node *node, struct rb_root_cached *root, const struct rb_augment_callbacks *augment) { - struct rb_node *rebalance = __rb_erase_augmented(node, &root->rb_root, - &root->rb_leftmost, - augment); - if (rebalance) - __rb_erase_color(rebalance, &root->rb_root, augment->rotate); + if (root->rb_leftmost == node) + root->rb_leftmost = rb_next(node); + rb_erase_augmented(node, &root->rb_root, augment); } #endif /* _LINUX_RBTREE_AUGMENTED_H */ diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index 04d04709f2bd..16ad66683ad0 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -100,7 +100,9 @@ struct fw_rsc_hdr { * the remote processor will be writing logs. * @RSC_VDEV: declare support for a virtio device, and serve as its * virtio header. - * @RSC_LAST: just keep this one at the end + * @RSC_LAST: just keep this one at the end of standard resources + * @RSC_VENDOR_START: start of the vendor specific resource types range + * @RSC_VENDOR_END: end of the vendor specific resource types range * * For more details regarding a specific resource type, please see its * dedicated structure below. @@ -111,11 +113,13 @@ struct fw_rsc_hdr { * please update it as needed. */ enum fw_resource_type { - RSC_CARVEOUT = 0, - RSC_DEVMEM = 1, - RSC_TRACE = 2, - RSC_VDEV = 3, - RSC_LAST = 4, + RSC_CARVEOUT = 0, + RSC_DEVMEM = 1, + RSC_TRACE = 2, + RSC_VDEV = 3, + RSC_LAST = 4, + RSC_VENDOR_START = 128, + RSC_VENDOR_END = 512, }; #define FW_RSC_ADDR_ANY (-1) @@ -340,12 +344,26 @@ struct rproc_mem_entry { struct firmware; /** + * enum rsc_handling_status - return status of rproc_ops handle_rsc hook + * @RSC_HANDLED: resource was handled + * @RSC_IGNORED: resource was ignored + */ +enum rsc_handling_status { + RSC_HANDLED = 0, + RSC_IGNORED = 1, +}; + +/** * struct rproc_ops - platform-specific device handlers * @start: power on the device and boot it * @stop: power off the device * @kick: kick a virtqueue (virtqueue id given as a parameter) * @da_to_va: optional platform hook to perform address translations * @parse_fw: parse firmware to extract information (e.g. resource table) + * @handle_rsc: optional platform hook to handle vendor resources. Should return + * RSC_HANDLED if resource was handled, RSC_IGNORED if not handled and a + * negative value on error + * @load_rsc_table: load resource table from firmware image * @find_loaded_rsc_table: find the loaded resouce table * @load: load firmware to memory, where the remote processor * expects to find it @@ -358,6 +376,8 @@ struct rproc_ops { void (*kick)(struct rproc *rproc, int vqid); void * (*da_to_va)(struct rproc *rproc, u64 da, int len); int (*parse_fw)(struct rproc *rproc, const struct firmware *fw); + int (*handle_rsc)(struct rproc *rproc, u32 rsc_type, void *rsc, + int offset, int avail); struct resource_table *(*find_loaded_rsc_table)( struct rproc *rproc, const struct firmware *fw); int (*load)(struct rproc *rproc, const struct firmware *fw); diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 532458698bde..efd8ce7675ed 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -15,10 +15,10 @@ */ struct sighand_struct { - refcount_t count; - struct k_sigaction action[_NSIG]; spinlock_t siglock; + refcount_t count; wait_queue_head_t signalfd_wqh; + struct k_sigaction action[_NSIG]; }; /* @@ -420,7 +420,6 @@ void task_join_group_stop(struct task_struct *task); static inline void set_restore_sigmask(void) { set_thread_flag(TIF_RESTORE_SIGMASK); - WARN_ON(!test_thread_flag(TIF_SIGPENDING)); } static inline void clear_tsk_restore_sigmask(struct task_struct *task) @@ -451,7 +450,6 @@ static inline bool test_and_clear_restore_sigmask(void) static inline void set_restore_sigmask(void) { current->restore_sigmask = true; - WARN_ON(!test_thread_flag(TIF_SIGPENDING)); } static inline void clear_tsk_restore_sigmask(struct task_struct *task) { @@ -484,6 +482,16 @@ static inline void restore_saved_sigmask(void) __set_current_blocked(¤t->saved_sigmask); } +extern int set_user_sigmask(const sigset_t __user *umask, size_t sigsetsize); + +static inline void restore_saved_sigmask_unless(bool interrupted) +{ + if (interrupted) + WARN_ON(!test_thread_flag(TIF_SIGPENDING)); + else + restore_saved_sigmask(); +} + static inline sigset_t *sigmask_to_save(void) { sigset_t *res = ¤t->blocked; diff --git a/include/linux/signal.h b/include/linux/signal.h index 78c2bb376954..b5d99482d3fe 100644 --- a/include/linux/signal.h +++ b/include/linux/signal.h @@ -273,10 +273,6 @@ extern int group_send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p, enum pid_type type); extern int __group_send_sig_info(int, struct kernel_siginfo *, struct task_struct *); extern int sigprocmask(int, sigset_t *, sigset_t *); -extern int set_user_sigmask(const sigset_t __user *usigmask, sigset_t *set, - sigset_t *oldset, size_t sigsetsize); -extern void restore_user_sigmask(const void __user *usigmask, - sigset_t *sigsaved, bool interrupted); extern void set_current_blocked(sigset_t *); extern void __set_current_blocked(const sigset_t *); extern int show_unhandled_signals; diff --git a/include/linux/soc/qcom/mdt_loader.h b/include/linux/soc/qcom/mdt_loader.h index 944b06aefb0f..e600baec6825 100644 --- a/include/linux/soc/qcom/mdt_loader.h +++ b/include/linux/soc/qcom/mdt_loader.h @@ -21,4 +21,6 @@ int qcom_mdt_load_no_init(struct device *dev, const struct firmware *fw, const char *fw_name, int pas_id, void *mem_region, phys_addr_t mem_phys, size_t mem_size, phys_addr_t *reloc_base); +void *qcom_mdt_read_metadata(const struct firmware *fw, size_t *data_len); + #endif diff --git a/include/linux/soc/ti/ti_sci_protocol.h b/include/linux/soc/ti/ti_sci_protocol.h index 568722a041bf..406e6717d252 100644 --- a/include/linux/soc/ti/ti_sci_protocol.h +++ b/include/linux/soc/ti/ti_sci_protocol.h @@ -166,29 +166,29 @@ struct ti_sci_dev_ops { * managed by driver for that purpose. */ struct ti_sci_clk_ops { - int (*get_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid, + int (*get_clock)(const struct ti_sci_handle *handle, u32 did, u32 cid, bool needs_ssc, bool can_change_freq, bool enable_input_term); - int (*idle_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid); - int (*put_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid); - int (*is_auto)(const struct ti_sci_handle *handle, u32 did, u8 cid, + int (*idle_clock)(const struct ti_sci_handle *handle, u32 did, u32 cid); + int (*put_clock)(const struct ti_sci_handle *handle, u32 did, u32 cid); + int (*is_auto)(const struct ti_sci_handle *handle, u32 did, u32 cid, bool *req_state); - int (*is_on)(const struct ti_sci_handle *handle, u32 did, u8 cid, + int (*is_on)(const struct ti_sci_handle *handle, u32 did, u32 cid, bool *req_state, bool *current_state); - int (*is_off)(const struct ti_sci_handle *handle, u32 did, u8 cid, + int (*is_off)(const struct ti_sci_handle *handle, u32 did, u32 cid, bool *req_state, bool *current_state); - int (*set_parent)(const struct ti_sci_handle *handle, u32 did, u8 cid, - u8 parent_id); - int (*get_parent)(const struct ti_sci_handle *handle, u32 did, u8 cid, - u8 *parent_id); + int (*set_parent)(const struct ti_sci_handle *handle, u32 did, u32 cid, + u32 parent_id); + int (*get_parent)(const struct ti_sci_handle *handle, u32 did, u32 cid, + u32 *parent_id); int (*get_num_parents)(const struct ti_sci_handle *handle, u32 did, - u8 cid, u8 *num_parents); + u32 cid, u32 *num_parents); int (*get_best_match_freq)(const struct ti_sci_handle *handle, u32 did, - u8 cid, u64 min_freq, u64 target_freq, + u32 cid, u64 min_freq, u64 target_freq, u64 max_freq, u64 *match_freq); - int (*set_freq)(const struct ti_sci_handle *handle, u32 did, u8 cid, + int (*set_freq)(const struct ti_sci_handle *handle, u32 did, u32 cid, u64 min_freq, u64 target_freq, u64 max_freq); - int (*get_freq)(const struct ti_sci_handle *handle, u32 did, u8 cid, + int (*get_freq)(const struct ti_sci_handle *handle, u32 did, u32 cid, u64 *current_freq); }; diff --git a/include/linux/sudmac.h b/include/linux/sudmac.h deleted file mode 100644 index cccc0a665d26..000000000000 --- a/include/linux/sudmac.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Header for the SUDMAC driver - * - * Copyright (C) 2013 Renesas Solutions Corp. - */ -#ifndef SUDMAC_H -#define SUDMAC_H - -#include <linux/dmaengine.h> -#include <linux/shdma-base.h> -#include <linux/types.h> - -/* Used by slave DMA clients to request DMA to/from a specific peripheral */ -struct sudmac_slave { - struct shdma_slave shdma_slave; /* Set by the platform */ -}; - -/* - * Supplied by platforms to specify, how a DMA channel has to be configured for - * a certain peripheral - */ -struct sudmac_slave_config { - int slave_id; -}; - -struct sudmac_channel { - unsigned long offset; - unsigned long config; - unsigned long wait; /* The configuable range is 0 to 3 */ - unsigned long dint_end_bit; -}; - -struct sudmac_pdata { - const struct sudmac_slave_config *slave; - int slave_num; - const struct sudmac_channel *channel; - int channel_num; -}; - -/* Definitions for the sudmac_channel.config */ -#define SUDMAC_TX_BUFFER_MODE BIT(0) -#define SUDMAC_RX_END_MODE BIT(1) - -/* Definitions for the sudmac_channel.dint_end_bit */ -#define SUDMAC_DMA_BIT_CH0 BIT(0) -#define SUDMAC_DMA_BIT_CH1 BIT(1) - -#endif diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 15bdb6fe71e5..877fd239b6ff 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -6,6 +6,8 @@ #include <linux/bug.h> #include <linux/mm_types.h> +#ifdef CONFIG_MMU + /* * swapcache pages are stored in the swapper_space radix tree. We want to * get good packing density in that tree, so the index should be dense in @@ -50,13 +52,11 @@ static inline pgoff_t swp_offset(swp_entry_t entry) return entry.val & SWP_OFFSET_MASK; } -#ifdef CONFIG_MMU /* check whether a pte points to a swap entry */ static inline int is_swap_pte(pte_t pte) { return !pte_none(pte) && !pte_present(pte); } -#endif /* * Convert the arch-dependent pte representation of a swp_entry_t into an @@ -360,4 +360,5 @@ static inline int non_swap_entry(swp_entry_t entry) } #endif +#endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 15a4ca5d7099..681047f8cc05 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -251,7 +251,7 @@ struct thermal_bind_params { * platform characterization. This value is relative to the * rest of the weights so a cooling device whose weight is * double that of another cooling device is twice as - * effective. See Documentation/thermal/sysfs-api.txt for more + * effective. See Documentation/thermal/sysfs-api.rst for more * information. */ int weight; @@ -259,7 +259,7 @@ struct thermal_bind_params { /* * This is a bit mask that gives the binding relation between this * thermal zone and cdev, for a particular trip point. - * See Documentation/thermal/sysfs-api.txt for more information. + * See Documentation/thermal/sysfs-api.rst for more information. */ int trip_mask; diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 8a62731673f7..5150436783e8 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -142,6 +142,7 @@ enum print_line_t { enum print_line_t trace_handle_return(struct trace_seq *s); void tracing_generic_entry_update(struct trace_entry *entry, + unsigned short type, unsigned long flags, int pc); struct trace_event_file; @@ -317,6 +318,14 @@ trace_event_name(struct trace_event_call *call) return call->name; } +static inline struct list_head * +trace_get_fields(struct trace_event_call *event_call) +{ + if (!event_call->class->get_fields) + return &event_call->class->fields; + return event_call->class->get_fields(event_call); +} + struct trace_array; struct trace_subsystem_dir; diff --git a/include/linux/tracehook.h b/include/linux/tracehook.h index 8446573cc682..36fb3bbed6b2 100644 --- a/include/linux/tracehook.h +++ b/include/linux/tracehook.h @@ -54,13 +54,15 @@ struct linux_binprm; /* * ptrace report for syscall entry and exit looks identical. */ -static inline int ptrace_report_syscall(struct pt_regs *regs) +static inline int ptrace_report_syscall(struct pt_regs *regs, + unsigned long message) { int ptrace = current->ptrace; if (!(ptrace & PT_PTRACED)) return 0; + current->ptrace_message = message; ptrace_notify(SIGTRAP | ((ptrace & PT_TRACESYSGOOD) ? 0x80 : 0)); /* @@ -73,6 +75,7 @@ static inline int ptrace_report_syscall(struct pt_regs *regs) current->exit_code = 0; } + current->ptrace_message = 0; return fatal_signal_pending(current); } @@ -98,7 +101,7 @@ static inline int ptrace_report_syscall(struct pt_regs *regs) static inline __must_check int tracehook_report_syscall_entry( struct pt_regs *regs) { - return ptrace_report_syscall(regs); + return ptrace_report_syscall(regs, PTRACE_EVENTMSG_SYSCALL_ENTRY); } /** @@ -123,7 +126,7 @@ static inline void tracehook_report_syscall_exit(struct pt_regs *regs, int step) if (step) user_single_step_report(regs); else - ptrace_report_syscall(regs); + ptrace_report_syscall(regs, PTRACE_EVENTMSG_SYSCALL_EXIT); } /** diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 2b70130af585..34a038563d97 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -203,7 +203,10 @@ static inline void pagefault_enable(void) /* * Is the pagefault handler disabled? If so, user access methods will not sleep. */ -#define pagefault_disabled() (current->pagefault_disabled != 0) +static inline bool pagefault_disabled(void) +{ + return current->pagefault_disabled != 0; +} /* * The pagefault handler is in general disabled by pagefault_disable() or @@ -240,6 +243,18 @@ extern long probe_kernel_read(void *dst, const void *src, size_t size); extern long __probe_kernel_read(void *dst, const void *src, size_t size); /* + * probe_user_read(): safely attempt to read from a location in user space + * @dst: pointer to the buffer that shall take the data + * @src: address to read from + * @size: size of the data chunk + * + * Safely read from address @src to the buffer at @dst. If a kernel fault + * happens, handle that and return -EFAULT. + */ +extern long probe_user_read(void *dst, const void __user *src, size_t size); +extern long __probe_user_read(void *dst, const void __user *src, size_t size); + +/* * probe_kernel_write(): safely attempt to write to a location * @dst: address to write to * @src: pointer to the data that shall be written @@ -252,6 +267,9 @@ extern long notrace probe_kernel_write(void *dst, const void *src, size_t size); extern long notrace __probe_kernel_write(void *dst, const void *src, size_t size); extern long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count); +extern long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr, + long count); +extern long strnlen_unsafe_user(const void __user *unsafe_addr, long count); /** * probe_kernel_address(): safely attempt to read from a location diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h index 8f46ff3449d5..871993696c5f 100644 --- a/include/sound/hda_codec.h +++ b/include/sound/hda_codec.h @@ -252,6 +252,8 @@ struct hda_codec { unsigned int auto_runtime_pm:1; /* enable automatic codec runtime pm */ unsigned int force_pin_prefix:1; /* Add location prefix */ unsigned int link_down_at_suspend:1; /* link down at runtime suspend */ + unsigned int relaxed_resume:1; /* don't resume forcibly for jack */ + #ifdef CONFIG_PM unsigned long power_on_acct; unsigned long power_off_acct; diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index f9eff010fc7e..2f6a669408bb 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -29,6 +29,7 @@ struct btrfs_qgroup_extent_record; struct btrfs_qgroup; struct extent_io_tree; struct prelim_ref; +struct btrfs_space_info; TRACE_DEFINE_ENUM(FLUSH_DELAYED_ITEMS_NR); TRACE_DEFINE_ENUM(FLUSH_DELAYED_ITEMS); @@ -2091,6 +2092,45 @@ DEFINE_BTRFS_LOCK_EVENT(btrfs_try_tree_read_lock); DEFINE_BTRFS_LOCK_EVENT(btrfs_try_tree_write_lock); DEFINE_BTRFS_LOCK_EVENT(btrfs_tree_read_lock_atomic); +DECLARE_EVENT_CLASS(btrfs__space_info_update, + + TP_PROTO(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *sinfo, u64 old, s64 diff), + + TP_ARGS(fs_info, sinfo, old, diff), + + TP_STRUCT__entry_btrfs( + __field( u64, type ) + __field( u64, old ) + __field( s64, diff ) + ), + + TP_fast_assign_btrfs(fs_info, + __entry->type = sinfo->flags; + __entry->old = old; + __entry->diff = diff; + ), + TP_printk_btrfs("type=%s old=%llu diff=%lld", + __print_flags(__entry->type, "|", BTRFS_GROUP_FLAGS), + __entry->old, __entry->diff) +); + +DEFINE_EVENT(btrfs__space_info_update, update_bytes_may_use, + + TP_PROTO(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *sinfo, u64 old, s64 diff), + + TP_ARGS(fs_info, sinfo, old, diff) +); + +DEFINE_EVENT(btrfs__space_info_update, update_bytes_pinned, + + TP_PROTO(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *sinfo, u64 old, s64 diff), + + TP_ARGS(fs_info, sinfo, old, diff) +); + #endif /* _TRACE_BTRFS_H */ /* This part must be outside protection */ diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h index abd238d0f7a4..63b1f506ea67 100644 --- a/include/uapi/asm-generic/mman-common.h +++ b/include/uapi/asm-generic/mman-common.h @@ -19,15 +19,18 @@ #define MAP_TYPE 0x0f /* Mask for type of mapping */ #define MAP_FIXED 0x10 /* Interpret addr exactly */ #define MAP_ANONYMOUS 0x20 /* don't use a file */ -#ifdef CONFIG_MMAP_ALLOW_UNINITIALIZED -# define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be uninitialized */ -#else -# define MAP_UNINITIALIZED 0x0 /* Don't support this flag */ -#endif -/* 0x0100 - 0x80000 flags are defined in asm-generic/mman.h */ +/* 0x0100 - 0x4000 flags are defined in asm-generic/mman.h */ +#define MAP_POPULATE 0x008000 /* populate (prefault) pagetables */ +#define MAP_NONBLOCK 0x010000 /* do not block on IO */ +#define MAP_STACK 0x020000 /* give out an address that is best suited for process/thread stacks */ +#define MAP_HUGETLB 0x040000 /* create a huge page mapping */ +#define MAP_SYNC 0x080000 /* perform synchronous page faults for the mapping */ #define MAP_FIXED_NOREPLACE 0x100000 /* MAP_FIXED which doesn't unmap underlying mapping */ +#define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be + * uninitialized */ + /* * Flags for mlock */ diff --git a/include/uapi/asm-generic/mman.h b/include/uapi/asm-generic/mman.h index 653687d9771b..57e8195d0b53 100644 --- a/include/uapi/asm-generic/mman.h +++ b/include/uapi/asm-generic/mman.h @@ -9,13 +9,11 @@ #define MAP_EXECUTABLE 0x1000 /* mark it as an executable */ #define MAP_LOCKED 0x2000 /* pages are locked */ #define MAP_NORESERVE 0x4000 /* don't check for reservations */ -#define MAP_POPULATE 0x8000 /* populate (prefault) pagetables */ -#define MAP_NONBLOCK 0x10000 /* do not block on IO */ -#define MAP_STACK 0x20000 /* give out an address that is best suited for process/thread stacks */ -#define MAP_HUGETLB 0x40000 /* create a huge page mapping */ -#define MAP_SYNC 0x80000 /* perform synchronous page faults for the mapping */ -/* Bits [26:31] are reserved, see mman-common.h for MAP_HUGETLB usage */ +/* + * Bits [26:31] are reserved, see asm-generic/hugetlb_encode.h + * for MAP_HUGETLB usage + */ #define MCL_CURRENT 1 /* lock all current mappings */ #define MCL_FUTURE 2 /* lock all future mappings */ diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h index 421239b98db2..34d5b34286fa 100644 --- a/include/uapi/linux/btrfs_tree.h +++ b/include/uapi/linux/btrfs_tree.h @@ -866,6 +866,8 @@ enum btrfs_raid_types { #define BTRFS_BLOCK_GROUP_RAID56_MASK (BTRFS_BLOCK_GROUP_RAID5 | \ BTRFS_BLOCK_GROUP_RAID6) +#define BTRFS_BLOCK_GROUP_RAID1_MASK (BTRFS_BLOCK_GROUP_RAID1) + /* * We need a bit for restriper to be able to tell when chunks of type * SINGLE are available. This "extended" profile format is used in diff --git a/include/uapi/linux/coda.h b/include/uapi/linux/coda.h index 695fade33c64..aa34c2dcae8d 100644 --- a/include/uapi/linux/coda.h +++ b/include/uapi/linux/coda.h @@ -86,10 +86,6 @@ typedef unsigned long long u_quad_t; #define inline -struct timespec { - long ts_sec; - long ts_nsec; -}; #else /* DJGPP but not KERNEL */ #include <sys/time.h> typedef unsigned long long u_quad_t; @@ -110,13 +106,6 @@ typedef unsigned long long u_quad_t; #define cdev_t dev_t #endif -#ifdef __CYGWIN32__ -struct timespec { - time_t tv_sec; /* seconds */ - long tv_nsec; /* nanoseconds */ -}; -#endif - #ifndef __BIT_TYPES_DEFINED__ #define __BIT_TYPES_DEFINED__ typedef signed char int8_t; @@ -211,6 +200,11 @@ struct CodaFid { */ enum coda_vtype { C_VNON, C_VREG, C_VDIR, C_VBLK, C_VCHR, C_VLNK, C_VSOCK, C_VFIFO, C_VBAD }; +struct coda_timespec { + int64_t tv_sec; /* seconds */ + long tv_nsec; /* nanoseconds */ +}; + struct coda_vattr { long va_type; /* vnode type (for create) */ u_short va_mode; /* files access mode and type */ @@ -220,9 +214,9 @@ struct coda_vattr { long va_fileid; /* file id */ u_quad_t va_size; /* file size in bytes */ long va_blocksize; /* blocksize preferred for i/o */ - struct timespec va_atime; /* time of last access */ - struct timespec va_mtime; /* time of last modification */ - struct timespec va_ctime; /* time file changed */ + struct coda_timespec va_atime; /* time of last access */ + struct coda_timespec va_mtime; /* time of last modification */ + struct coda_timespec va_ctime; /* time file changed */ u_long va_gen; /* generation number of file */ u_long va_flags; /* flags defined for file */ cdev_t va_rdev; /* device special file represents */ @@ -277,7 +271,8 @@ struct coda_statfs { #define CODA_STATFS 34 #define CODA_STORE 35 #define CODA_RELEASE 36 -#define CODA_NCALLS 37 +#define CODA_ACCESS_INTENT 37 +#define CODA_NCALLS 38 #define DOWNCALL(opcode) (opcode >= CODA_REPLACE && opcode <= CODA_PURGEFID) @@ -287,7 +282,12 @@ struct coda_statfs { #define CIOC_KERNEL_VERSION _IOWR('c', 10, size_t) -#define CODA_KERNEL_VERSION 3 /* 128-bit file identifiers */ +// CODA_KERNEL_VERSION 0 /* don't care about kernel version number */ +// CODA_KERNEL_VERSION 1 /* The old venus 4.6 compatible interface */ +// CODA_KERNEL_VERSION 2 /* venus_lookup gets an extra parameter */ +// CODA_KERNEL_VERSION 3 /* 128-bit file identifiers */ +// CODA_KERNEL_VERSION 4 /* 64-bit timespec */ +#define CODA_KERNEL_VERSION 5 /* access intent support */ /* * Venus <-> Coda RPC arguments @@ -295,8 +295,8 @@ struct coda_statfs { struct coda_in_hdr { u_int32_t opcode; u_int32_t unique; /* Keep multiple outstanding msgs distinct */ - pid_t pid; - pid_t pgid; + __kernel_pid_t pid; + __kernel_pid_t pgid; vuid_t uid; }; @@ -642,6 +642,25 @@ struct coda_statfs_out { struct coda_statfs stat; }; +#define CODA_ACCESS_TYPE_READ 1 +#define CODA_ACCESS_TYPE_WRITE 2 +#define CODA_ACCESS_TYPE_MMAP 3 +#define CODA_ACCESS_TYPE_READ_FINISH 4 +#define CODA_ACCESS_TYPE_WRITE_FINISH 5 + +/* coda_access_intent: NO_OUT */ +struct coda_access_intent_in { + struct coda_in_hdr ih; + struct CodaFid VFid; + int count; + int pos; + int type; +}; + +struct coda_access_intent_out { + struct coda_out_hdr out; +}; + /* * Occasionally, we don't cache the fid returned by CODA_LOOKUP. * For instance, if the fid is inconsistent. @@ -673,6 +692,7 @@ union inputArgs { struct coda_open_by_fd_in coda_open_by_fd; struct coda_open_by_path_in coda_open_by_path; struct coda_statfs_in coda_statfs; + struct coda_access_intent_in coda_access_intent; }; union outputArgs { diff --git a/include/uapi/linux/coda_psdev.h b/include/uapi/linux/coda_psdev.h deleted file mode 100644 index aa6623efd2dd..000000000000 --- a/include/uapi/linux/coda_psdev.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI__CODA_PSDEV_H -#define _UAPI__CODA_PSDEV_H - -#include <linux/magic.h> - -#define CODA_PSDEV_MAJOR 67 -#define MAX_CODADEVS 5 /* how many do we allow */ - - -/* messages between coda filesystem in kernel and Venus */ -struct upc_req { - struct list_head uc_chain; - caddr_t uc_data; - u_short uc_flags; - u_short uc_inSize; /* Size is at most 5000 bytes */ - u_short uc_outSize; - u_short uc_opcode; /* copied from data to save lookup */ - int uc_unique; - wait_queue_head_t uc_sleep; /* process' wait queue */ -}; - -#define CODA_REQ_ASYNC 0x1 -#define CODA_REQ_READ 0x2 -#define CODA_REQ_WRITE 0x4 -#define CODA_REQ_ABORT 0x8 - -#endif /* _UAPI__CODA_PSDEV_H */ diff --git a/include/uapi/linux/ptrace.h b/include/uapi/linux/ptrace.h index d5a1b8a492b9..a71b6e3b03eb 100644 --- a/include/uapi/linux/ptrace.h +++ b/include/uapi/linux/ptrace.h @@ -73,6 +73,41 @@ struct seccomp_metadata { __u64 flags; /* Output: filter's flags */ }; +#define PTRACE_GET_SYSCALL_INFO 0x420e +#define PTRACE_SYSCALL_INFO_NONE 0 +#define PTRACE_SYSCALL_INFO_ENTRY 1 +#define PTRACE_SYSCALL_INFO_EXIT 2 +#define PTRACE_SYSCALL_INFO_SECCOMP 3 + +struct ptrace_syscall_info { + __u8 op; /* PTRACE_SYSCALL_INFO_* */ + __u32 arch __attribute__((__aligned__(sizeof(__u32)))); + __u64 instruction_pointer; + __u64 stack_pointer; + union { + struct { + __u64 nr; + __u64 args[6]; + } entry; + struct { + __s64 rval; + __u8 is_error; + } exit; + struct { + __u64 nr; + __u64 args[6]; + __u32 ret_data; + } seccomp; + }; +}; + +/* + * These values are stored in task->ptrace_message + * by tracehook_report_syscall_* to describe the current syscall-stop. + */ +#define PTRACE_EVENTMSG_SYSCALL_ENTRY 1 +#define PTRACE_EVENTMSG_SYSCALL_EXIT 2 + /* Read signals from a shared (process wide) queue */ #define PTRACE_PEEKSIGINFO_SHARED (1 << 0) diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h index 6d5c3b2d4f4d..348fd0176f75 100644 --- a/include/uapi/linux/virtio_ids.h +++ b/include/uapi/linux/virtio_ids.h @@ -43,5 +43,7 @@ #define VIRTIO_ID_INPUT 18 /* virtio input */ #define VIRTIO_ID_VSOCK 19 /* virtio vsock transport */ #define VIRTIO_ID_CRYPTO 20 /* virtio crypto */ +#define VIRTIO_ID_IOMMU 23 /* virtio IOMMU */ +#define VIRTIO_ID_PMEM 27 /* virtio pmem */ #endif /* _LINUX_VIRTIO_IDS_H */ diff --git a/include/uapi/linux/virtio_iommu.h b/include/uapi/linux/virtio_iommu.h new file mode 100644 index 000000000000..ba1b460c9944 --- /dev/null +++ b/include/uapi/linux/virtio_iommu.h @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Virtio-iommu definition v0.9 + * + * Copyright (C) 2018 Arm Ltd. + */ +#ifndef _UAPI_LINUX_VIRTIO_IOMMU_H +#define _UAPI_LINUX_VIRTIO_IOMMU_H + +#include <linux/types.h> + +/* Feature bits */ +#define VIRTIO_IOMMU_F_INPUT_RANGE 0 +#define VIRTIO_IOMMU_F_DOMAIN_BITS 1 +#define VIRTIO_IOMMU_F_MAP_UNMAP 2 +#define VIRTIO_IOMMU_F_BYPASS 3 +#define VIRTIO_IOMMU_F_PROBE 4 + +struct virtio_iommu_range { + __u64 start; + __u64 end; +}; + +struct virtio_iommu_config { + /* Supported page sizes */ + __u64 page_size_mask; + /* Supported IOVA range */ + struct virtio_iommu_range input_range; + /* Max domain ID size */ + __u8 domain_bits; + __u8 padding[3]; + /* Probe buffer size */ + __u32 probe_size; +}; + +/* Request types */ +#define VIRTIO_IOMMU_T_ATTACH 0x01 +#define VIRTIO_IOMMU_T_DETACH 0x02 +#define VIRTIO_IOMMU_T_MAP 0x03 +#define VIRTIO_IOMMU_T_UNMAP 0x04 +#define VIRTIO_IOMMU_T_PROBE 0x05 + +/* Status types */ +#define VIRTIO_IOMMU_S_OK 0x00 +#define VIRTIO_IOMMU_S_IOERR 0x01 +#define VIRTIO_IOMMU_S_UNSUPP 0x02 +#define VIRTIO_IOMMU_S_DEVERR 0x03 +#define VIRTIO_IOMMU_S_INVAL 0x04 +#define VIRTIO_IOMMU_S_RANGE 0x05 +#define VIRTIO_IOMMU_S_NOENT 0x06 +#define VIRTIO_IOMMU_S_FAULT 0x07 + +struct virtio_iommu_req_head { + __u8 type; + __u8 reserved[3]; +}; + +struct virtio_iommu_req_tail { + __u8 status; + __u8 reserved[3]; +}; + +struct virtio_iommu_req_attach { + struct virtio_iommu_req_head head; + __le32 domain; + __le32 endpoint; + __u8 reserved[8]; + struct virtio_iommu_req_tail tail; +}; + +struct virtio_iommu_req_detach { + struct virtio_iommu_req_head head; + __le32 domain; + __le32 endpoint; + __u8 reserved[8]; + struct virtio_iommu_req_tail tail; +}; + +#define VIRTIO_IOMMU_MAP_F_READ (1 << 0) +#define VIRTIO_IOMMU_MAP_F_WRITE (1 << 1) +#define VIRTIO_IOMMU_MAP_F_EXEC (1 << 2) +#define VIRTIO_IOMMU_MAP_F_MMIO (1 << 3) + +#define VIRTIO_IOMMU_MAP_F_MASK (VIRTIO_IOMMU_MAP_F_READ | \ + VIRTIO_IOMMU_MAP_F_WRITE | \ + VIRTIO_IOMMU_MAP_F_EXEC | \ + VIRTIO_IOMMU_MAP_F_MMIO) + +struct virtio_iommu_req_map { + struct virtio_iommu_req_head head; + __le32 domain; + __le64 virt_start; + __le64 virt_end; + __le64 phys_start; + __le32 flags; + struct virtio_iommu_req_tail tail; +}; + +struct virtio_iommu_req_unmap { + struct virtio_iommu_req_head head; + __le32 domain; + __le64 virt_start; + __le64 virt_end; + __u8 reserved[4]; + struct virtio_iommu_req_tail tail; +}; + +#define VIRTIO_IOMMU_PROBE_T_NONE 0 +#define VIRTIO_IOMMU_PROBE_T_RESV_MEM 1 + +#define VIRTIO_IOMMU_PROBE_T_MASK 0xfff + +struct virtio_iommu_probe_property { + __le16 type; + __le16 length; +}; + +#define VIRTIO_IOMMU_RESV_MEM_T_RESERVED 0 +#define VIRTIO_IOMMU_RESV_MEM_T_MSI 1 + +struct virtio_iommu_probe_resv_mem { + struct virtio_iommu_probe_property head; + __u8 subtype; + __u8 reserved[3]; + __le64 start; + __le64 end; +}; + +struct virtio_iommu_req_probe { + struct virtio_iommu_req_head head; + __le32 endpoint; + __u8 reserved[64]; + + __u8 properties[]; + + /* + * Tail follows the variable-length properties array. No padding, + * property lengths are all aligned on 8 bytes. + */ +}; + +/* Fault types */ +#define VIRTIO_IOMMU_FAULT_R_UNKNOWN 0 +#define VIRTIO_IOMMU_FAULT_R_DOMAIN 1 +#define VIRTIO_IOMMU_FAULT_R_MAPPING 2 + +#define VIRTIO_IOMMU_FAULT_F_READ (1 << 0) +#define VIRTIO_IOMMU_FAULT_F_WRITE (1 << 1) +#define VIRTIO_IOMMU_FAULT_F_EXEC (1 << 2) +#define VIRTIO_IOMMU_FAULT_F_ADDRESS (1 << 8) + +struct virtio_iommu_fault { + __u8 reason; + __u8 reserved[3]; + __le32 flags; + __le32 endpoint; + __u8 reserved2[4]; + __le64 address; +}; + +#endif diff --git a/include/uapi/linux/virtio_pmem.h b/include/uapi/linux/virtio_pmem.h new file mode 100644 index 000000000000..9a63ed6d062f --- /dev/null +++ b/include/uapi/linux/virtio_pmem.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ +/* + * Definitions for virtio-pmem devices. + * + * Copyright (C) 2019 Red Hat, Inc. + * + * Author(s): Pankaj Gupta <pagupta@redhat.com> + */ + +#ifndef _UAPI_LINUX_VIRTIO_PMEM_H +#define _UAPI_LINUX_VIRTIO_PMEM_H + +#include <linux/types.h> +#include <linux/virtio_ids.h> +#include <linux/virtio_config.h> + +struct virtio_pmem_config { + __u64 start; + __u64 size; +}; + +#define VIRTIO_PMEM_REQ_TYPE_FLUSH 0 + +struct virtio_pmem_resp { + /* Host return status corresponding to flush request */ + __le32 ret; +}; + +struct virtio_pmem_req { + /* command type */ + __le32 type; +}; + +#endif diff --git a/init/Kconfig b/init/Kconfig index 381cdfee6e0e..bd7d650d4a99 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1827,7 +1827,7 @@ config SLAB_FREELIST_HARDENED help Many kernel heap attacks try to target slab cache metadata and other infrastructure. This options makes minor performance - sacrifies to harden the kernel slab allocator against common + sacrifices to harden the kernel slab allocator against common freelist exploit methods. config SHUFFLE_PAGE_ALLOCATOR @@ -1859,7 +1859,7 @@ config SLUB_CPU_PARTIAL depends on SLUB && SMP bool "SLUB per cpu partial cache" help - Per cpu partial caches accellerate objects allocation and freeing + Per cpu partial caches accelerate objects allocation and freeing that is local to a processor at the price of more indeterminism in the latency of the free. On overflow these caches will be cleared which requires the taking of locks that may cause latency spikes. diff --git a/ipc/mqueue.c b/ipc/mqueue.c index 216cad1ff0d0..65c351564ad0 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -438,7 +438,6 @@ static void mqueue_evict_inode(struct inode *inode) { struct mqueue_inode_info *info; struct user_struct *user; - unsigned long mq_bytes, mq_treesize; struct ipc_namespace *ipc_ns; struct msg_msg *msg, *nmsg; LIST_HEAD(tmp_msg); @@ -461,16 +460,18 @@ static void mqueue_evict_inode(struct inode *inode) free_msg(msg); } - /* Total amount of bytes accounted for the mqueue */ - mq_treesize = info->attr.mq_maxmsg * sizeof(struct msg_msg) + - min_t(unsigned int, info->attr.mq_maxmsg, MQ_PRIO_MAX) * - sizeof(struct posix_msg_tree_node); - - mq_bytes = mq_treesize + (info->attr.mq_maxmsg * - info->attr.mq_msgsize); - user = info->user; if (user) { + unsigned long mq_bytes, mq_treesize; + + /* Total amount of bytes accounted for the mqueue */ + mq_treesize = info->attr.mq_maxmsg * sizeof(struct msg_msg) + + min_t(unsigned int, info->attr.mq_maxmsg, MQ_PRIO_MAX) * + sizeof(struct posix_msg_tree_node); + + mq_bytes = mq_treesize + (info->attr.mq_maxmsg * + info->attr.mq_msgsize); + spin_lock(&mq_lock); user->mq_bytes -= mq_bytes; /* diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 62fa5a82a065..9de232229063 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -129,15 +129,17 @@ setup_io_tlb_npages(char *str) } early_param("swiotlb", setup_io_tlb_npages); +static bool no_iotlb_memory; + unsigned long swiotlb_nr_tbl(void) { - return io_tlb_nslabs; + return unlikely(no_iotlb_memory) ? 0 : io_tlb_nslabs; } EXPORT_SYMBOL_GPL(swiotlb_nr_tbl); unsigned int swiotlb_max_segment(void) { - return max_segment; + return unlikely(no_iotlb_memory) ? 0 : max_segment; } EXPORT_SYMBOL_GPL(swiotlb_max_segment); @@ -160,8 +162,6 @@ unsigned long swiotlb_size_or_default(void) return size ? size : (IO_TLB_DEFAULT_SIZE); } -static bool no_iotlb_memory; - void swiotlb_print_info(void) { unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT; @@ -317,6 +317,14 @@ swiotlb_late_init_with_default_size(size_t default_size) return rc; } +static void swiotlb_cleanup(void) +{ + io_tlb_end = 0; + io_tlb_start = 0; + io_tlb_nslabs = 0; + max_segment = 0; +} + int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs) { @@ -367,10 +375,7 @@ cleanup4: sizeof(int))); io_tlb_list = NULL; cleanup3: - io_tlb_end = 0; - io_tlb_start = 0; - io_tlb_nslabs = 0; - max_segment = 0; + swiotlb_cleanup(); return -ENOMEM; } @@ -394,10 +399,7 @@ void __init swiotlb_exit(void) memblock_free_late(io_tlb_start, PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT)); } - io_tlb_start = 0; - io_tlb_end = 0; - io_tlb_nslabs = 0; - max_segment = 0; + swiotlb_cleanup(); } /* @@ -546,7 +548,7 @@ not_found: if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n", size, io_tlb_nslabs, tmp_io_tlb_used); - return DMA_MAPPING_ERROR; + return (phys_addr_t)DMA_MAPPING_ERROR; found: io_tlb_used += nslots; spin_unlock_irqrestore(&io_tlb_lock, flags); @@ -664,7 +666,7 @@ bool swiotlb_map(struct device *dev, phys_addr_t *phys, dma_addr_t *dma_addr, /* Oh well, have to allocate and map a bounce buffer. */ *phys = swiotlb_tbl_map_single(dev, __phys_to_dma(dev, io_tlb_start), *phys, size, dir, attrs); - if (*phys == DMA_MAPPING_ERROR) + if (*phys == (phys_addr_t)DMA_MAPPING_ERROR) return false; /* Ensure that the address returned is DMA'ble */ diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 9f5433a52488..9873fc627d61 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -2276,6 +2276,7 @@ static int __init init_kprobes(void) init_test_probes(); return err; } +subsys_initcall(init_kprobes); #ifdef CONFIG_DEBUG_FS static void report_probe(struct seq_file *pi, struct kprobe *p, @@ -2588,5 +2589,3 @@ static int __init debugfs_kprobe_init(void) late_initcall(debugfs_kprobe_init); #endif /* CONFIG_DEBUG_FS */ - -module_init(init_kprobes); diff --git a/kernel/module.c b/kernel/module.c index a2cee14a83f3..5933395af9a0 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1492,8 +1492,7 @@ static void add_sect_attrs(struct module *mod, const struct load_info *info) for (i = 0; i < info->hdr->e_shnum; i++) if (!sect_empty(&info->sechdrs[i])) nloaded++; - size[0] = ALIGN(sizeof(*sect_attrs) - + nloaded * sizeof(sect_attrs->attrs[0]), + size[0] = ALIGN(struct_size(sect_attrs, attrs, nloaded), sizeof(sect_attrs->grp.attrs[0])); size[1] = (nloaded + 1) * sizeof(sect_attrs->grp.attrs[0]); sect_attrs = kzalloc(size[0] + size[1], GFP_KERNEL); @@ -1697,6 +1696,8 @@ static int add_usage_links(struct module *mod) return ret; } +static void module_remove_modinfo_attrs(struct module *mod, int end); + static int module_add_modinfo_attrs(struct module *mod) { struct module_attribute *attr; @@ -1711,24 +1712,34 @@ static int module_add_modinfo_attrs(struct module *mod) return -ENOMEM; temp_attr = mod->modinfo_attrs; - for (i = 0; (attr = modinfo_attrs[i]) && !error; i++) { + for (i = 0; (attr = modinfo_attrs[i]); i++) { if (!attr->test || attr->test(mod)) { memcpy(temp_attr, attr, sizeof(*temp_attr)); sysfs_attr_init(&temp_attr->attr); error = sysfs_create_file(&mod->mkobj.kobj, &temp_attr->attr); + if (error) + goto error_out; ++temp_attr; } } + + return 0; + +error_out: + if (i > 0) + module_remove_modinfo_attrs(mod, --i); return error; } -static void module_remove_modinfo_attrs(struct module *mod) +static void module_remove_modinfo_attrs(struct module *mod, int end) { struct module_attribute *attr; int i; for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) { + if (end >= 0 && i > end) + break; /* pick a field to test for end of list */ if (!attr->attr.name) break; @@ -1816,7 +1827,7 @@ static int mod_sysfs_setup(struct module *mod, return 0; out_unreg_modinfo_attrs: - module_remove_modinfo_attrs(mod); + module_remove_modinfo_attrs(mod, -1); out_unreg_param: module_param_sysfs_remove(mod); out_unreg_holders: @@ -1852,7 +1863,7 @@ static void mod_sysfs_fini(struct module *mod) { } -static void module_remove_modinfo_attrs(struct module *mod) +static void module_remove_modinfo_attrs(struct module *mod, int end) { } @@ -1868,14 +1879,14 @@ static void init_param_lock(struct module *mod) static void mod_sysfs_teardown(struct module *mod) { del_usage_links(mod); - module_remove_modinfo_attrs(mod); + module_remove_modinfo_attrs(mod, -1); module_param_sysfs_remove(mod); kobject_put(mod->mkobj.drivers_dir); kobject_put(mod->holders_dir); mod_sysfs_fini(mod); } -#ifdef CONFIG_STRICT_MODULE_RWX +#ifdef CONFIG_ARCH_HAS_STRICT_MODULE_RWX /* * LKM RO/NX protection: protect module's text/ro-data * from modification and any data from execution. @@ -1898,6 +1909,7 @@ static void frob_text(const struct module_layout *layout, layout->text_size >> PAGE_SHIFT); } +#ifdef CONFIG_STRICT_MODULE_RWX static void frob_rodata(const struct module_layout *layout, int (*set_memory)(unsigned long start, int num_pages)) { @@ -1949,13 +1961,9 @@ void module_enable_ro(const struct module *mod, bool after_init) set_vm_flush_reset_perms(mod->core_layout.base); set_vm_flush_reset_perms(mod->init_layout.base); frob_text(&mod->core_layout, set_memory_ro); - frob_text(&mod->core_layout, set_memory_x); frob_rodata(&mod->core_layout, set_memory_ro); - frob_text(&mod->init_layout, set_memory_ro); - frob_text(&mod->init_layout, set_memory_x); - frob_rodata(&mod->init_layout, set_memory_ro); if (after_init) @@ -2014,9 +2022,19 @@ void set_all_modules_text_ro(void) } mutex_unlock(&module_mutex); } -#else +#else /* !CONFIG_STRICT_MODULE_RWX */ static void module_enable_nx(const struct module *mod) { } -#endif +#endif /* CONFIG_STRICT_MODULE_RWX */ +static void module_enable_x(const struct module *mod) +{ + frob_text(&mod->core_layout, set_memory_x); + frob_text(&mod->init_layout, set_memory_x); +} +#else /* !CONFIG_ARCH_HAS_STRICT_MODULE_RWX */ +static void module_enable_nx(const struct module *mod) { } +static void module_enable_x(const struct module *mod) { } +#endif /* CONFIG_ARCH_HAS_STRICT_MODULE_RWX */ + #ifdef CONFIG_LIVEPATCH /* @@ -2723,6 +2741,11 @@ void * __weak module_alloc(unsigned long size) return vmalloc_exec(size); } +bool __weak module_exit_section(const char *name) +{ + return strstarts(name, ".exit"); +} + #ifdef CONFIG_DEBUG_KMEMLEAK static void kmemleak_load_module(const struct module *mod, const struct load_info *info) @@ -2912,7 +2935,7 @@ static int rewrite_section_headers(struct load_info *info, int flags) #ifndef CONFIG_MODULE_UNLOAD /* Don't load .exit sections */ - if (strstarts(info->secstrings+shdr->sh_name, ".exit")) + if (module_exit_section(info->secstrings+shdr->sh_name)) shdr->sh_flags &= ~(unsigned long)SHF_ALLOC; #endif } @@ -3390,8 +3413,7 @@ static bool finished_loading(const char *name) sched_annotate_sleep(); mutex_lock(&module_mutex); mod = find_module_all(name, strlen(name), true); - ret = !mod || mod->state == MODULE_STATE_LIVE - || mod->state == MODULE_STATE_GOING; + ret = !mod || mod->state == MODULE_STATE_LIVE; mutex_unlock(&module_mutex); return ret; @@ -3581,8 +3603,7 @@ again: mutex_lock(&module_mutex); old = find_module_all(mod->name, strlen(mod->name), true); if (old != NULL) { - if (old->state == MODULE_STATE_COMING - || old->state == MODULE_STATE_UNFORMED) { + if (old->state != MODULE_STATE_LIVE) { /* Wait in case it fails to load. */ mutex_unlock(&module_mutex); err = wait_event_interruptible(module_wq, @@ -3621,6 +3642,7 @@ static int complete_formation(struct module *mod, struct load_info *info) module_enable_ro(mod, false); module_enable_nx(mod); + module_enable_x(mod); /* Mark state as coming so strong_try_module_get() ignores us, * but kallsyms etc. can see us. */ diff --git a/kernel/pid.c b/kernel/pid.c index 16263b526560..0a9f2e437217 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -37,14 +37,14 @@ #include <linux/init_task.h> #include <linux/syscalls.h> #include <linux/proc_ns.h> -#include <linux/proc_fs.h> +#include <linux/refcount.h> #include <linux/anon_inodes.h> #include <linux/sched/signal.h> #include <linux/sched/task.h> #include <linux/idr.h> struct pid init_struct_pid = { - .count = ATOMIC_INIT(1), + .count = REFCOUNT_INIT(1), .tasks = { { .first = NULL }, { .first = NULL }, @@ -108,8 +108,7 @@ void put_pid(struct pid *pid) return; ns = pid->numbers[pid->level].ns; - if ((atomic_read(&pid->count) == 1) || - atomic_dec_and_test(&pid->count)) { + if (refcount_dec_and_test(&pid->count)) { kmem_cache_free(ns->pid_cachep, pid); put_pid_ns(ns); } @@ -212,7 +211,7 @@ struct pid *alloc_pid(struct pid_namespace *ns) } get_pid_ns(ns); - atomic_set(&pid->count, 1); + refcount_set(&pid->count, 1); for (type = 0; type < PIDTYPE_MAX; ++type) INIT_HLIST_HEAD(&pid->tasks[type]); diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 83a531cea2f3..cb9ddcc08119 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -32,6 +32,8 @@ #include <linux/compat.h> #include <linux/sched/signal.h> +#include <asm/syscall.h> /* for syscall_get_* */ + /* * Access another process' address space via ptrace. * Source/target buffer must be kernel space, @@ -897,7 +899,100 @@ static int ptrace_regset(struct task_struct *task, int req, unsigned int type, * to ensure no machine forgets it. */ EXPORT_SYMBOL_GPL(task_user_regset_view); -#endif + +static unsigned long +ptrace_get_syscall_info_entry(struct task_struct *child, struct pt_regs *regs, + struct ptrace_syscall_info *info) +{ + unsigned long args[ARRAY_SIZE(info->entry.args)]; + int i; + + info->op = PTRACE_SYSCALL_INFO_ENTRY; + info->entry.nr = syscall_get_nr(child, regs); + syscall_get_arguments(child, regs, args); + for (i = 0; i < ARRAY_SIZE(args); i++) + info->entry.args[i] = args[i]; + + /* args is the last field in struct ptrace_syscall_info.entry */ + return offsetofend(struct ptrace_syscall_info, entry.args); +} + +static unsigned long +ptrace_get_syscall_info_seccomp(struct task_struct *child, struct pt_regs *regs, + struct ptrace_syscall_info *info) +{ + /* + * As struct ptrace_syscall_info.entry is currently a subset + * of struct ptrace_syscall_info.seccomp, it makes sense to + * initialize that subset using ptrace_get_syscall_info_entry(). + * This can be reconsidered in the future if these structures + * diverge significantly enough. + */ + ptrace_get_syscall_info_entry(child, regs, info); + info->op = PTRACE_SYSCALL_INFO_SECCOMP; + info->seccomp.ret_data = child->ptrace_message; + + /* ret_data is the last field in struct ptrace_syscall_info.seccomp */ + return offsetofend(struct ptrace_syscall_info, seccomp.ret_data); +} + +static unsigned long +ptrace_get_syscall_info_exit(struct task_struct *child, struct pt_regs *regs, + struct ptrace_syscall_info *info) +{ + info->op = PTRACE_SYSCALL_INFO_EXIT; + info->exit.rval = syscall_get_error(child, regs); + info->exit.is_error = !!info->exit.rval; + if (!info->exit.is_error) + info->exit.rval = syscall_get_return_value(child, regs); + + /* is_error is the last field in struct ptrace_syscall_info.exit */ + return offsetofend(struct ptrace_syscall_info, exit.is_error); +} + +static int +ptrace_get_syscall_info(struct task_struct *child, unsigned long user_size, + void __user *datavp) +{ + struct pt_regs *regs = task_pt_regs(child); + struct ptrace_syscall_info info = { + .op = PTRACE_SYSCALL_INFO_NONE, + .arch = syscall_get_arch(child), + .instruction_pointer = instruction_pointer(regs), + .stack_pointer = user_stack_pointer(regs), + }; + unsigned long actual_size = offsetof(struct ptrace_syscall_info, entry); + unsigned long write_size; + + /* + * This does not need lock_task_sighand() to access + * child->last_siginfo because ptrace_freeze_traced() + * called earlier by ptrace_check_attach() ensures that + * the tracee cannot go away and clear its last_siginfo. + */ + switch (child->last_siginfo ? child->last_siginfo->si_code : 0) { + case SIGTRAP | 0x80: + switch (child->ptrace_message) { + case PTRACE_EVENTMSG_SYSCALL_ENTRY: + actual_size = ptrace_get_syscall_info_entry(child, regs, + &info); + break; + case PTRACE_EVENTMSG_SYSCALL_EXIT: + actual_size = ptrace_get_syscall_info_exit(child, regs, + &info); + break; + } + break; + case SIGTRAP | (PTRACE_EVENT_SECCOMP << 8): + actual_size = ptrace_get_syscall_info_seccomp(child, regs, + &info); + break; + } + + write_size = min(actual_size, user_size); + return copy_to_user(datavp, &info, write_size) ? -EFAULT : actual_size; +} +#endif /* CONFIG_HAVE_ARCH_TRACEHOOK */ int ptrace_request(struct task_struct *child, long request, unsigned long addr, unsigned long data) @@ -1114,6 +1209,10 @@ int ptrace_request(struct task_struct *child, long request, ret = __put_user(kiov.iov_len, &uiov->iov_len); break; } + + case PTRACE_GET_SYSCALL_INFO: + ret = ptrace_get_syscall_info(child, addr, datavp); + break; #endif case PTRACE_SECCOMP_GET_FILTER: diff --git a/kernel/signal.c b/kernel/signal.c index dabe100d2091..91b789dd6e72 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -2951,80 +2951,49 @@ EXPORT_SYMBOL(sigprocmask); * * This is useful for syscalls such as ppoll, pselect, io_pgetevents and * epoll_pwait where a new sigmask is passed from userland for the syscalls. + * + * Note that it does set_restore_sigmask() in advance, so it must be always + * paired with restore_saved_sigmask_unless() before return from syscall. */ -int set_user_sigmask(const sigset_t __user *usigmask, sigset_t *set, - sigset_t *oldset, size_t sigsetsize) +int set_user_sigmask(const sigset_t __user *umask, size_t sigsetsize) { - if (!usigmask) - return 0; + sigset_t kmask; + if (!umask) + return 0; if (sigsetsize != sizeof(sigset_t)) return -EINVAL; - if (copy_from_user(set, usigmask, sizeof(sigset_t))) + if (copy_from_user(&kmask, umask, sizeof(sigset_t))) return -EFAULT; - *oldset = current->blocked; - set_current_blocked(set); + set_restore_sigmask(); + current->saved_sigmask = current->blocked; + set_current_blocked(&kmask); return 0; } -EXPORT_SYMBOL(set_user_sigmask); #ifdef CONFIG_COMPAT -int set_compat_user_sigmask(const compat_sigset_t __user *usigmask, - sigset_t *set, sigset_t *oldset, +int set_compat_user_sigmask(const compat_sigset_t __user *umask, size_t sigsetsize) { - if (!usigmask) - return 0; + sigset_t kmask; + if (!umask) + return 0; if (sigsetsize != sizeof(compat_sigset_t)) return -EINVAL; - if (get_compat_sigset(set, usigmask)) + if (get_compat_sigset(&kmask, umask)) return -EFAULT; - *oldset = current->blocked; - set_current_blocked(set); + set_restore_sigmask(); + current->saved_sigmask = current->blocked; + set_current_blocked(&kmask); return 0; } -EXPORT_SYMBOL(set_compat_user_sigmask); #endif -/* - * restore_user_sigmask: - * usigmask: sigmask passed in from userland. - * sigsaved: saved sigmask when the syscall started and changed the sigmask to - * usigmask. - * - * This is useful for syscalls such as ppoll, pselect, io_pgetevents and - * epoll_pwait where a new sigmask is passed in from userland for the syscalls. - */ -void restore_user_sigmask(const void __user *usigmask, sigset_t *sigsaved, - bool interrupted) -{ - - if (!usigmask) - return; - /* - * When signals are pending, do not restore them here. - * Restoring sigmask here can lead to delivering signals that the above - * syscalls are intended to block because of the sigmask passed in. - */ - if (interrupted) { - current->saved_sigmask = *sigsaved; - set_restore_sigmask(); - return; - } - - /* - * This is needed because the fast syscall return path does not restore - * saved_sigmask when signals are not pending. - */ - set_current_blocked(sigsaved); -} -EXPORT_SYMBOL(restore_user_sigmask); - /** * sys_rt_sigprocmask - change the list of currently blocked signals * @how: whether to add, remove, or set signals diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 1c1ad1e14f21..43186ccfa139 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -188,17 +188,17 @@ extern int no_unaligned_warning; * enum sysctl_writes_mode - supported sysctl write modes * * @SYSCTL_WRITES_LEGACY: each write syscall must fully contain the sysctl value - * to be written, and multiple writes on the same sysctl file descriptor - * will rewrite the sysctl value, regardless of file position. No warning - * is issued when the initial position is not 0. + * to be written, and multiple writes on the same sysctl file descriptor + * will rewrite the sysctl value, regardless of file position. No warning + * is issued when the initial position is not 0. * @SYSCTL_WRITES_WARN: same as above but warn when the initial file position is - * not 0. + * not 0. * @SYSCTL_WRITES_STRICT: writes to numeric sysctl entries must always be at - * file position 0 and the value must be fully contained in the buffer - * sent to the write syscall. If dealing with strings respect the file - * position, but restrict this to the max length of the buffer, anything - * passed the max lenght will be ignored. Multiple writes will append - * to the buffer. + * file position 0 and the value must be fully contained in the buffer + * sent to the write syscall. If dealing with strings respect the file + * position, but restrict this to the max length of the buffer, anything + * passed the max length will be ignored. Multiple writes will append + * to the buffer. * * These write modes control how current file position affects the behavior of * updating sysctl values through the proc interface on each write. diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 564e5fdb025f..98da8998c25c 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -597,9 +597,19 @@ config FTRACE_STARTUP_TEST functioning properly. It will do tests on all the configured tracers of ftrace. +config EVENT_TRACE_STARTUP_TEST + bool "Run selftest on trace events" + depends on FTRACE_STARTUP_TEST + default y + help + This option performs a test on all trace events in the system. + It basically just enables each event and runs some code that + will trigger events (not necessarily the event it enables) + This may take some time run as there are a lot of events. + config EVENT_TRACE_TEST_SYSCALLS bool "Run selftest on syscall events" - depends on FTRACE_STARTUP_TEST + depends on EVENT_TRACE_STARTUP_TEST help This option will also enable testing every syscall event. It only enables the event and disables it and runs various loads diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 576c41644e77..eca34503f178 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1622,6 +1622,11 @@ static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) return keep_regs; } +static struct ftrace_ops * +ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); +static struct ftrace_ops * +ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); + static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, int filter_hash, bool inc) @@ -1750,15 +1755,17 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, } /* - * If the rec had TRAMP enabled, then it needs to - * be cleared. As TRAMP can only be enabled iff - * there is only a single ops attached to it. - * In otherwords, always disable it on decrementing. - * In the future, we may set it if rec count is - * decremented to one, and the ops that is left - * has a trampoline. + * The TRAMP needs to be set only if rec count + * is decremented to one, and the ops that is + * left has a trampoline. As TRAMP can only be + * enabled if there is only a single ops attached + * to it. */ - rec->flags &= ~FTRACE_FL_TRAMP; + if (ftrace_rec_count(rec) == 1 && + ftrace_find_tramp_ops_any(rec)) + rec->flags |= FTRACE_FL_TRAMP; + else + rec->flags &= ~FTRACE_FL_TRAMP; /* * flags will be cleared in ftrace_check_record() @@ -1768,7 +1775,7 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, count++; /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */ - update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE; + update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE; /* Shortcut, if we handled all records, we are done. */ if (!all && count == hash->count) @@ -1951,11 +1958,6 @@ static void print_ip_ins(const char *fmt, const unsigned char *p) printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); } -static struct ftrace_ops * -ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); -static struct ftrace_ops * -ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); - enum ftrace_bug_type ftrace_bug_type; const void *ftrace_expected; @@ -2047,7 +2049,7 @@ void ftrace_bug(int failed, struct dyn_ftrace *rec) } } -static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) +static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) { unsigned long flag = 0UL; @@ -2146,28 +2148,28 @@ static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) /** * ftrace_update_record, set a record that now is tracing or not * @rec: the record to update - * @enable: set to 1 if the record is tracing, zero to force disable + * @enable: set to true if the record is tracing, false to force disable * * The records that represent all functions that can be traced need * to be updated when tracing has been enabled. */ -int ftrace_update_record(struct dyn_ftrace *rec, int enable) +int ftrace_update_record(struct dyn_ftrace *rec, bool enable) { - return ftrace_check_record(rec, enable, 1); + return ftrace_check_record(rec, enable, true); } /** * ftrace_test_record, check if the record has been enabled or not * @rec: the record to test - * @enable: set to 1 to check if enabled, 0 if it is disabled + * @enable: set to true to check if enabled, false if it is disabled * * The arch code may need to test if a record is already set to * tracing to determine how to modify the function code that it * represents. */ -int ftrace_test_record(struct dyn_ftrace *rec, int enable) +int ftrace_test_record(struct dyn_ftrace *rec, bool enable) { - return ftrace_check_record(rec, enable, 0); + return ftrace_check_record(rec, enable, false); } static struct ftrace_ops * @@ -2356,7 +2358,7 @@ unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) } static int -__ftrace_replace_code(struct dyn_ftrace *rec, int enable) +__ftrace_replace_code(struct dyn_ftrace *rec, bool enable) { unsigned long ftrace_old_addr; unsigned long ftrace_addr; @@ -2395,7 +2397,7 @@ void __weak ftrace_replace_code(int mod_flags) { struct dyn_ftrace *rec; struct ftrace_page *pg; - int enable = mod_flags & FTRACE_MODIFY_ENABLE_FL; + bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL; int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL; int failed; diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 05b0b3139ebc..66358d66c933 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -128,16 +128,7 @@ int ring_buffer_print_entry_header(struct trace_seq *s) #define RB_ALIGNMENT 4U #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ - -#ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS -# define RB_FORCE_8BYTE_ALIGNMENT 0 -# define RB_ARCH_ALIGNMENT RB_ALIGNMENT -#else -# define RB_FORCE_8BYTE_ALIGNMENT 1 -# define RB_ARCH_ALIGNMENT 8U -#endif - -#define RB_ALIGN_DATA __aligned(RB_ARCH_ALIGNMENT) +#define RB_ALIGN_DATA __aligned(RB_ALIGNMENT) /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX @@ -2373,7 +2364,7 @@ rb_update_event(struct ring_buffer_per_cpu *cpu_buffer, event->time_delta = delta; length -= RB_EVNT_HDR_SIZE; - if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) { + if (length > RB_MAX_SMALL_DATA) { event->type_len = 0; event->array[0] = length; } else @@ -2388,11 +2379,11 @@ static unsigned rb_calculate_event_length(unsigned length) if (!length) length++; - if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) + if (length > RB_MAX_SMALL_DATA) length += sizeof(event.array[0]); length += RB_EVNT_HDR_SIZE; - length = ALIGN(length, RB_ARCH_ALIGNMENT); + length = ALIGN(length, RB_ALIGNMENT); /* * In case the time delta is larger than the 27 bits for it diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index c90c687cf950..525a97fbbc60 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -366,7 +366,7 @@ trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct } /** - * trace_pid_filter_add_remove_task - Add or remove a task from a pid_list + * trace_filter_add_remove_task - Add or remove a task from a pid_list * @pid_list: The list to modify * @self: The current task for fork or NULL for exit * @task: The task to add or remove @@ -743,8 +743,7 @@ trace_event_setup(struct ring_buffer_event *event, { struct trace_entry *ent = ring_buffer_event_data(event); - tracing_generic_entry_update(ent, flags, pc); - ent->type = type; + tracing_generic_entry_update(ent, type, flags, pc); } static __always_inline struct ring_buffer_event * @@ -2312,13 +2311,14 @@ enum print_line_t trace_handle_return(struct trace_seq *s) EXPORT_SYMBOL_GPL(trace_handle_return); void -tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, - int pc) +tracing_generic_entry_update(struct trace_entry *entry, unsigned short type, + unsigned long flags, int pc) { struct task_struct *tsk = current; entry->preempt_count = pc & 0xff; entry->pid = (tsk) ? tsk->pid : 0; + entry->type = type; entry->flags = #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | @@ -4842,12 +4842,13 @@ static const char readme_msg[] = "\t args: <name>=fetcharg[:type]\n" "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n" #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API - "\t $stack<index>, $stack, $retval, $comm, $arg<N>\n" + "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n" #else - "\t $stack<index>, $stack, $retval, $comm\n" + "\t $stack<index>, $stack, $retval, $comm,\n" #endif + "\t +|-[u]<offset>(<fetcharg>)\n" "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n" - "\t b<bit-width>@<bit-offset>/<container-size>,\n" + "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n" "\t <type>\\[<array-size>\\]\n" #ifdef CONFIG_HIST_TRIGGERS "\t field: <stype> <name>;\n" diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index 4629a6104474..0892e38ed6fb 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c @@ -416,8 +416,7 @@ void perf_trace_buf_update(void *record, u16 type) unsigned long flags; local_save_flags(flags); - tracing_generic_entry_update(entry, flags, pc); - entry->type = type; + tracing_generic_entry_update(entry, type, flags, pc); } NOKPROBE_SYMBOL(perf_trace_buf_update); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 0ce3db67f556..c7506bc81b75 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -70,14 +70,6 @@ static int system_refcount_dec(struct event_subsystem *system) #define while_for_each_event_file() \ } -static struct list_head * -trace_get_fields(struct trace_event_call *event_call) -{ - if (!event_call->class->get_fields) - return &event_call->class->fields; - return event_call->class->get_fields(event_call); -} - static struct ftrace_event_field * __find_event_field(struct list_head *head, char *name) { @@ -3190,7 +3182,7 @@ void __init trace_event_init(void) event_trace_enable(); } -#ifdef CONFIG_FTRACE_STARTUP_TEST +#ifdef CONFIG_EVENT_TRACE_STARTUP_TEST static DEFINE_SPINLOCK(test_spinlock); static DEFINE_SPINLOCK(test_spinlock_irq); diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 5079d1db3754..c773b8fb270c 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -1084,6 +1084,9 @@ int filter_assign_type(const char *type) if (strchr(type, '[') && strstr(type, "char")) return FILTER_STATIC_STRING; + if (strcmp(type, "char *") == 0 || strcmp(type, "const char *") == 0) + return FILTER_PTR_STRING; + return FILTER_OTHER; } diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 7d736248a070..9d483ad9bb6c 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -12,6 +12,8 @@ #include <linux/rculist.h> #include <linux/error-injection.h> +#include <asm/setup.h> /* for COMMAND_LINE_SIZE */ + #include "trace_dynevent.h" #include "trace_kprobe_selftest.h" #include "trace_probe.h" @@ -19,6 +21,18 @@ #define KPROBE_EVENT_SYSTEM "kprobes" #define KRETPROBE_MAXACTIVE_MAX 4096 +#define MAX_KPROBE_CMDLINE_SIZE 1024 + +/* Kprobe early definition from command line */ +static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; +static bool kprobe_boot_events_enabled __initdata; + +static int __init set_kprobe_boot_events(char *str) +{ + strlcpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE); + return 0; +} +__setup("kprobe_event=", set_kprobe_boot_events); static int trace_kprobe_create(int argc, const char **argv); static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); @@ -128,8 +142,8 @@ static bool trace_kprobe_match(const char *system, const char *event, { struct trace_kprobe *tk = to_trace_kprobe(ev); - return strcmp(trace_event_name(&tk->tp.call), event) == 0 && - (!system || strcmp(tk->tp.call.class->system, system) == 0); + return strcmp(trace_probe_name(&tk->tp), event) == 0 && + (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0); } static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) @@ -143,6 +157,12 @@ static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) return nhit; } +static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) +{ + return !(list_empty(&tk->rp.kp.list) && + hlist_unhashed(&tk->rp.kp.hlist)); +} + /* Return 0 if it fails to find the symbol address */ static nokprobe_inline unsigned long trace_kprobe_address(struct trace_kprobe *tk) @@ -183,6 +203,16 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); static int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs); +static void free_trace_kprobe(struct trace_kprobe *tk) +{ + if (tk) { + trace_probe_cleanup(&tk->tp); + kfree(tk->symbol); + free_percpu(tk->nhit); + kfree(tk); + } +} + /* * Allocate new trace_probe and initialize it (including kprobes). */ @@ -220,49 +250,20 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group, tk->rp.kp.pre_handler = kprobe_dispatcher; tk->rp.maxactive = maxactive; + INIT_HLIST_NODE(&tk->rp.kp.hlist); + INIT_LIST_HEAD(&tk->rp.kp.list); - if (!event || !group) { - ret = -EINVAL; - goto error; - } - - tk->tp.call.class = &tk->tp.class; - tk->tp.call.name = kstrdup(event, GFP_KERNEL); - if (!tk->tp.call.name) - goto error; - - tk->tp.class.system = kstrdup(group, GFP_KERNEL); - if (!tk->tp.class.system) + ret = trace_probe_init(&tk->tp, event, group); + if (ret < 0) goto error; dyn_event_init(&tk->devent, &trace_kprobe_ops); - INIT_LIST_HEAD(&tk->tp.files); return tk; error: - kfree(tk->tp.call.name); - kfree(tk->symbol); - free_percpu(tk->nhit); - kfree(tk); + free_trace_kprobe(tk); return ERR_PTR(ret); } -static void free_trace_kprobe(struct trace_kprobe *tk) -{ - int i; - - if (!tk) - return; - - for (i = 0; i < tk->tp.nr_args; i++) - traceprobe_free_probe_arg(&tk->tp.args[i]); - - kfree(tk->tp.call.class->system); - kfree(tk->tp.call.name); - kfree(tk->symbol); - free_percpu(tk->nhit); - kfree(tk); -} - static struct trace_kprobe *find_trace_kprobe(const char *event, const char *group) { @@ -270,8 +271,8 @@ static struct trace_kprobe *find_trace_kprobe(const char *event, struct trace_kprobe *tk; for_each_trace_kprobe(tk, pos) - if (strcmp(trace_event_name(&tk->tp.call), event) == 0 && - strcmp(tk->tp.call.class->system, group) == 0) + if (strcmp(trace_probe_name(&tk->tp), event) == 0 && + strcmp(trace_probe_group_name(&tk->tp), group) == 0) return tk; return NULL; } @@ -280,7 +281,7 @@ static inline int __enable_trace_kprobe(struct trace_kprobe *tk) { int ret = 0; - if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) { + if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { if (trace_kprobe_is_return(tk)) ret = enable_kretprobe(&tk->rp); else @@ -297,34 +298,27 @@ static inline int __enable_trace_kprobe(struct trace_kprobe *tk) static int enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) { - struct event_file_link *link; + bool enabled = trace_probe_is_enabled(&tk->tp); int ret = 0; if (file) { - link = kmalloc(sizeof(*link), GFP_KERNEL); - if (!link) { - ret = -ENOMEM; - goto out; - } - - link->file = file; - list_add_tail_rcu(&link->list, &tk->tp.files); + ret = trace_probe_add_file(&tk->tp, file); + if (ret) + return ret; + } else + trace_probe_set_flag(&tk->tp, TP_FLAG_PROFILE); - tk->tp.flags |= TP_FLAG_TRACE; - ret = __enable_trace_kprobe(tk); - if (ret) { - list_del_rcu(&link->list); - kfree(link); - tk->tp.flags &= ~TP_FLAG_TRACE; - } + if (enabled) + return 0; - } else { - tk->tp.flags |= TP_FLAG_PROFILE; - ret = __enable_trace_kprobe(tk); - if (ret) - tk->tp.flags &= ~TP_FLAG_PROFILE; + ret = __enable_trace_kprobe(tk); + if (ret) { + if (file) + trace_probe_remove_file(&tk->tp, file); + else + trace_probe_clear_flag(&tk->tp, TP_FLAG_PROFILE); } - out: + return ret; } @@ -335,54 +329,34 @@ enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) static int disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) { - struct event_file_link *link = NULL; - int wait = 0; + struct trace_probe *tp = &tk->tp; int ret = 0; if (file) { - link = find_event_file_link(&tk->tp, file); - if (!link) { - ret = -EINVAL; - goto out; - } - - list_del_rcu(&link->list); - wait = 1; - if (!list_empty(&tk->tp.files)) + if (!trace_probe_get_file_link(tp, file)) + return -ENOENT; + if (!trace_probe_has_single_file(tp)) goto out; - - tk->tp.flags &= ~TP_FLAG_TRACE; + trace_probe_clear_flag(tp, TP_FLAG_TRACE); } else - tk->tp.flags &= ~TP_FLAG_PROFILE; + trace_probe_clear_flag(tp, TP_FLAG_PROFILE); - if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) { + if (!trace_probe_is_enabled(tp) && trace_kprobe_is_registered(tk)) { if (trace_kprobe_is_return(tk)) disable_kretprobe(&tk->rp); else disable_kprobe(&tk->rp.kp); - wait = 1; } - /* - * if tk is not added to any list, it must be a local trace_kprobe - * created with perf_event_open. We don't need to wait for these - * trace_kprobes - */ - if (list_empty(&tk->devent.list)) - wait = 0; out: - if (wait) { + if (file) /* - * Synchronize with kprobe_trace_func/kretprobe_trace_func - * to ensure disabled (all running handlers are finished). - * This is not only for kfree(), but also the caller, - * trace_remove_event_call() supposes it for releasing - * event_call related objects, which will be accessed in - * the kprobe_trace_func/kretprobe_trace_func. + * Synchronization is done in below function. For perf event, + * file == NULL and perf_trace_event_unreg() calls + * tracepoint_synchronize_unregister() to ensure synchronize + * event. We don't need to care about it. */ - synchronize_rcu(); - kfree(link); /* Ignored if link == NULL */ - } + trace_probe_remove_file(tp, file); return ret; } @@ -415,7 +389,7 @@ static int __register_trace_kprobe(struct trace_kprobe *tk) { int i, ret; - if (trace_probe_is_registered(&tk->tp)) + if (trace_kprobe_is_registered(tk)) return -EINVAL; if (within_notrace_func(tk)) { @@ -441,21 +415,20 @@ static int __register_trace_kprobe(struct trace_kprobe *tk) else ret = register_kprobe(&tk->rp.kp); - if (ret == 0) - tk->tp.flags |= TP_FLAG_REGISTERED; return ret; } /* Internal unregister function - just handle k*probes and flags */ static void __unregister_trace_kprobe(struct trace_kprobe *tk) { - if (trace_probe_is_registered(&tk->tp)) { + if (trace_kprobe_is_registered(tk)) { if (trace_kprobe_is_return(tk)) unregister_kretprobe(&tk->rp); else unregister_kprobe(&tk->rp.kp); - tk->tp.flags &= ~TP_FLAG_REGISTERED; - /* Cleanup kprobe for reuse */ + /* Cleanup kprobe for reuse and mark it unregistered */ + INIT_HLIST_NODE(&tk->rp.kp.hlist); + INIT_LIST_HEAD(&tk->rp.kp.list); if (tk->rp.kp.symbol_name) tk->rp.kp.addr = NULL; } @@ -487,8 +460,8 @@ static int register_trace_kprobe(struct trace_kprobe *tk) mutex_lock(&event_mutex); /* Delete old (same name) event if exist */ - old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call), - tk->tp.call.class->system); + old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), + trace_probe_group_name(&tk->tp)); if (old_tk) { ret = unregister_trace_kprobe(old_tk); if (ret < 0) @@ -541,7 +514,7 @@ static int trace_kprobe_module_callback(struct notifier_block *nb, ret = __register_trace_kprobe(tk); if (ret) pr_warn("Failed to re-register probe %s on %s: %d\n", - trace_event_name(&tk->tp.call), + trace_probe_name(&tk->tp), mod->name, ret); } } @@ -716,6 +689,10 @@ static int trace_kprobe_create(int argc, const char *argv[]) goto error; /* This can be -ENOMEM */ } + ret = traceprobe_set_print_fmt(&tk->tp, is_return); + if (ret < 0) + goto error; + ret = register_trace_kprobe(tk); if (ret) { trace_probe_log_set_index(1); @@ -767,8 +744,8 @@ static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) int i; seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); - seq_printf(m, ":%s/%s", tk->tp.call.class->system, - trace_event_name(&tk->tp.call)); + seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), + trace_probe_name(&tk->tp)); if (!tk->symbol) seq_printf(m, " 0x%p", tk->rp.kp.addr); @@ -842,7 +819,7 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) tk = to_trace_kprobe(ev); seq_printf(m, " %-44s %15lu %15lu\n", - trace_event_name(&tk->tp.call), + trace_probe_name(&tk->tp), trace_kprobe_nhit(tk), tk->rp.kp.nmissed); @@ -886,6 +863,15 @@ fetch_store_strlen(unsigned long addr) return (ret < 0) ? ret : len; } +/* Return the length of string -- including null terminal byte */ +static nokprobe_inline int +fetch_store_strlen_user(unsigned long addr) +{ + const void __user *uaddr = (__force const void __user *)addr; + + return strnlen_unsafe_user(uaddr, MAX_STRING_SIZE); +} + /* * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max * length and relative data location. @@ -894,19 +880,46 @@ static nokprobe_inline int fetch_store_string(unsigned long addr, void *dest, void *base) { int maxlen = get_loc_len(*(u32 *)dest); - u8 *dst = get_loc_data(dest, base); + void *__dest; long ret; if (unlikely(!maxlen)) return -ENOMEM; + + __dest = get_loc_data(dest, base); + /* * Try to get string again, since the string can be changed while * probing. */ - ret = strncpy_from_unsafe(dst, (void *)addr, maxlen); + ret = strncpy_from_unsafe(__dest, (void *)addr, maxlen); + if (ret >= 0) + *(u32 *)dest = make_data_loc(ret, __dest - base); + + return ret; +} +/* + * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf + * with max length and relative data location. + */ +static nokprobe_inline int +fetch_store_string_user(unsigned long addr, void *dest, void *base) +{ + const void __user *uaddr = (__force const void __user *)addr; + int maxlen = get_loc_len(*(u32 *)dest); + void *__dest; + long ret; + + if (unlikely(!maxlen)) + return -ENOMEM; + + __dest = get_loc_data(dest, base); + + ret = strncpy_from_unsafe_user(__dest, uaddr, maxlen); if (ret >= 0) - *(u32 *)dest = make_data_loc(ret, (void *)dst - base); + *(u32 *)dest = make_data_loc(ret, __dest - base); + return ret; } @@ -916,6 +929,14 @@ probe_mem_read(void *dest, void *src, size_t size) return probe_kernel_read(dest, src, size); } +static nokprobe_inline int +probe_mem_read_user(void *dest, void *src, size_t size) +{ + const void __user *uaddr = (__force const void __user *)src; + + return probe_user_read(dest, uaddr, size); +} + /* Note that we don't verify it, since the code does not come from user space */ static int process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest, @@ -971,7 +992,7 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, struct ring_buffer *buffer; int size, dsize, pc; unsigned long irq_flags; - struct trace_event_call *call = &tk->tp.call; + struct trace_event_call *call = trace_probe_event_call(&tk->tp); WARN_ON(call != trace_file->event_call); @@ -1003,7 +1024,7 @@ kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) { struct event_file_link *link; - list_for_each_entry_rcu(link, &tk->tp.files, list) + trace_probe_for_each_link_rcu(link, &tk->tp) __kprobe_trace_func(tk, regs, link->file); } NOKPROBE_SYMBOL(kprobe_trace_func); @@ -1019,7 +1040,7 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, struct ring_buffer *buffer; int size, pc, dsize; unsigned long irq_flags; - struct trace_event_call *call = &tk->tp.call; + struct trace_event_call *call = trace_probe_event_call(&tk->tp); WARN_ON(call != trace_file->event_call); @@ -1053,7 +1074,7 @@ kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, { struct event_file_link *link; - list_for_each_entry_rcu(link, &tk->tp.files, list) + trace_probe_for_each_link_rcu(link, &tk->tp) __kretprobe_trace_func(tk, ri, regs, link->file); } NOKPROBE_SYMBOL(kretprobe_trace_func); @@ -1070,7 +1091,7 @@ print_kprobe_event(struct trace_iterator *iter, int flags, field = (struct kprobe_trace_entry_head *)iter->ent; tp = container_of(event, struct trace_probe, call.event); - trace_seq_printf(s, "%s: (", trace_event_name(&tp->call)); + trace_seq_printf(s, "%s: (", trace_probe_name(tp)); if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) goto out; @@ -1097,7 +1118,7 @@ print_kretprobe_event(struct trace_iterator *iter, int flags, field = (struct kretprobe_trace_entry_head *)iter->ent; tp = container_of(event, struct trace_probe, call.event); - trace_seq_printf(s, "%s: (", trace_event_name(&tp->call)); + trace_seq_printf(s, "%s: (", trace_probe_name(tp)); if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) goto out; @@ -1149,7 +1170,7 @@ static int kretprobe_event_define_fields(struct trace_event_call *event_call) static int kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) { - struct trace_event_call *call = &tk->tp.call; + struct trace_event_call *call = trace_probe_event_call(&tk->tp); struct kprobe_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; @@ -1199,7 +1220,7 @@ static void kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, struct pt_regs *regs) { - struct trace_event_call *call = &tk->tp.call; + struct trace_event_call *call = trace_probe_event_call(&tk->tp); struct kretprobe_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; @@ -1299,10 +1320,10 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) raw_cpu_inc(*tk->nhit); - if (tk->tp.flags & TP_FLAG_TRACE) + if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) kprobe_trace_func(tk, regs); #ifdef CONFIG_PERF_EVENTS - if (tk->tp.flags & TP_FLAG_PROFILE) + if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) ret = kprobe_perf_func(tk, regs); #endif return ret; @@ -1316,10 +1337,10 @@ kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) raw_cpu_inc(*tk->nhit); - if (tk->tp.flags & TP_FLAG_TRACE) + if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) kretprobe_trace_func(tk, ri, regs); #ifdef CONFIG_PERF_EVENTS - if (tk->tp.flags & TP_FLAG_PROFILE) + if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) kretprobe_perf_func(tk, ri, regs); #endif return 0; /* We don't tweek kernel, so just return 0 */ @@ -1334,10 +1355,10 @@ static struct trace_event_functions kprobe_funcs = { .trace = print_kprobe_event }; -static inline void init_trace_event_call(struct trace_kprobe *tk, - struct trace_event_call *call) +static inline void init_trace_event_call(struct trace_kprobe *tk) { - INIT_LIST_HEAD(&call->class->fields); + struct trace_event_call *call = trace_probe_event_call(&tk->tp); + if (trace_kprobe_is_return(tk)) { call->event.funcs = &kretprobe_funcs; call->class->define_fields = kretprobe_event_define_fields; @@ -1353,37 +1374,14 @@ static inline void init_trace_event_call(struct trace_kprobe *tk, static int register_kprobe_event(struct trace_kprobe *tk) { - struct trace_event_call *call = &tk->tp.call; - int ret = 0; - - init_trace_event_call(tk, call); + init_trace_event_call(tk); - if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) - return -ENOMEM; - ret = register_trace_event(&call->event); - if (!ret) { - kfree(call->print_fmt); - return -ENODEV; - } - ret = trace_add_event_call(call); - if (ret) { - pr_info("Failed to register kprobe event: %s\n", - trace_event_name(call)); - kfree(call->print_fmt); - unregister_trace_event(&call->event); - } - return ret; + return trace_probe_register_event_call(&tk->tp); } static int unregister_kprobe_event(struct trace_kprobe *tk) { - int ret; - - /* tp->event is unregistered in trace_remove_event_call() */ - ret = trace_remove_event_call(&tk->tp.call); - if (!ret) - kfree(tk->tp.call.print_fmt); - return ret; + return trace_probe_unregister_event_call(&tk->tp); } #ifdef CONFIG_PERF_EVENTS @@ -1413,7 +1411,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs, return ERR_CAST(tk); } - init_trace_event_call(tk, &tk->tp.call); + init_trace_event_call(tk); if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) { ret = -ENOMEM; @@ -1421,12 +1419,10 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs, } ret = __register_trace_kprobe(tk); - if (ret < 0) { - kfree(tk->tp.call.print_fmt); + if (ret < 0) goto error; - } - return &tk->tp.call; + return trace_probe_event_call(&tk->tp); error: free_trace_kprobe(tk); return ERR_PTR(ret); @@ -1445,11 +1441,50 @@ void destroy_local_trace_kprobe(struct trace_event_call *event_call) __unregister_trace_kprobe(tk); - kfree(tk->tp.call.print_fmt); free_trace_kprobe(tk); } #endif /* CONFIG_PERF_EVENTS */ +static __init void enable_boot_kprobe_events(void) +{ + struct trace_array *tr = top_trace_array(); + struct trace_event_file *file; + struct trace_kprobe *tk; + struct dyn_event *pos; + + mutex_lock(&event_mutex); + for_each_trace_kprobe(tk, pos) { + list_for_each_entry(file, &tr->events, list) + if (file->event_call == trace_probe_event_call(&tk->tp)) + trace_event_enable_disable(file, 1, 0); + } + mutex_unlock(&event_mutex); +} + +static __init void setup_boot_kprobe_events(void) +{ + char *p, *cmd = kprobe_boot_events_buf; + int ret; + + strreplace(kprobe_boot_events_buf, ',', ' '); + + while (cmd && *cmd != '\0') { + p = strchr(cmd, ';'); + if (p) + *p++ = '\0'; + + ret = trace_run_command(cmd, create_or_delete_trace_kprobe); + if (ret) + pr_warn("Failed to add event(%d): %s\n", ret, cmd); + else + kprobe_boot_events_enabled = true; + + cmd = p; + } + + enable_boot_kprobe_events(); +} + /* Make a tracefs interface for controlling probe points */ static __init int init_kprobe_trace(void) { @@ -1481,6 +1516,9 @@ static __init int init_kprobe_trace(void) if (!entry) pr_warn("Could not create tracefs 'kprobe_profile' entry\n"); + + setup_boot_kprobe_events(); + return 0; } fs_initcall(init_kprobe_trace); @@ -1493,7 +1531,7 @@ find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) struct trace_event_file *file; list_for_each_entry(file, &tr->events, list) - if (file->event_call == &tk->tp.call) + if (file->event_call == trace_probe_event_call(&tk->tp)) return file; return NULL; @@ -1513,6 +1551,11 @@ static __init int kprobe_trace_self_tests_init(void) if (tracing_is_disabled()) return -ENODEV; + if (kprobe_boot_events_enabled) { + pr_info("Skipping kprobe tests due to kprobe_event on cmdline\n"); + return 0; + } + target = kprobe_trace_selftest_target; pr_info("Testing kprobe tracing: "); diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index a347faced959..dbef0d135075 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -78,6 +78,8 @@ static const struct fetch_type probe_fetch_types[] = { /* Special types */ __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, "__data_loc char[]"), + __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, + "__data_loc char[]"), /* Basic types */ ASSIGN_FETCH_TYPE(u8, u8, 0), ASSIGN_FETCH_TYPE(u16, u16, 0), @@ -322,6 +324,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type, { struct fetch_insn *code = *pcode; unsigned long param; + int deref = FETCH_OP_DEREF; long offset = 0; char *tmp; int ret = 0; @@ -394,9 +397,14 @@ parse_probe_arg(char *arg, const struct fetch_type *type, break; case '+': /* deref memory */ - arg++; /* Skip '+', because kstrtol() rejects it. */ - /* fall through */ case '-': + if (arg[1] == 'u') { + deref = FETCH_OP_UDEREF; + arg[1] = arg[0]; + arg++; + } + if (arg[0] == '+') + arg++; /* Skip '+', because kstrtol() rejects it. */ tmp = strchr(arg, '('); if (!tmp) { trace_probe_log_err(offs, DEREF_NEED_BRACE); @@ -432,7 +440,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type, } *pcode = code; - code->op = FETCH_OP_DEREF; + code->op = deref; code->offset = offset; } break; @@ -569,15 +577,17 @@ static int traceprobe_parse_probe_arg_body(char *arg, ssize_t *size, goto fail; /* Store operation */ - if (!strcmp(parg->type->name, "string")) { - if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_IMM && - code->op != FETCH_OP_COMM) { + if (!strcmp(parg->type->name, "string") || + !strcmp(parg->type->name, "ustring")) { + if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF && + code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM) { trace_probe_log_err(offset + (t ? (t - arg) : 0), BAD_STRING); ret = -EINVAL; goto fail; } - if (code->op != FETCH_OP_DEREF || parg->count) { + if ((code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM) || + parg->count) { /* * IMM and COMM is pointing actual address, those must * be kept, and if parg->count != 0, this is an array @@ -590,12 +600,20 @@ static int traceprobe_parse_probe_arg_body(char *arg, ssize_t *size, goto fail; } } - code->op = FETCH_OP_ST_STRING; /* In DEREF case, replace it */ + /* If op == DEREF, replace it with STRING */ + if (!strcmp(parg->type->name, "ustring") || + code->op == FETCH_OP_UDEREF) + code->op = FETCH_OP_ST_USTRING; + else + code->op = FETCH_OP_ST_STRING; code->size = parg->type->size; parg->dynamic = true; } else if (code->op == FETCH_OP_DEREF) { code->op = FETCH_OP_ST_MEM; code->size = parg->type->size; + } else if (code->op == FETCH_OP_UDEREF) { + code->op = FETCH_OP_ST_UMEM; + code->size = parg->type->size; } else { code++; if (code->op != FETCH_OP_NOP) { @@ -618,7 +636,8 @@ static int traceprobe_parse_probe_arg_body(char *arg, ssize_t *size, /* Loop(Array) operation */ if (parg->count) { if (scode->op != FETCH_OP_ST_MEM && - scode->op != FETCH_OP_ST_STRING) { + scode->op != FETCH_OP_ST_STRING && + scode->op != FETCH_OP_ST_USTRING) { trace_probe_log_err(offset + (t ? (t - arg) : 0), BAD_STRING); ret = -EINVAL; @@ -825,6 +844,7 @@ static int __set_print_fmt(struct trace_probe *tp, char *buf, int len, int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return) { + struct trace_event_call *call = trace_probe_event_call(tp); int len; char *print_fmt; @@ -836,7 +856,7 @@ int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return) /* Second: actually write the @print_fmt */ __set_print_fmt(tp, print_fmt, len + 1, is_return); - tp->call.print_fmt = print_fmt; + call->print_fmt = print_fmt; return 0; } @@ -865,3 +885,105 @@ int traceprobe_define_arg_fields(struct trace_event_call *event_call, } return 0; } + + +void trace_probe_cleanup(struct trace_probe *tp) +{ + struct trace_event_call *call = trace_probe_event_call(tp); + int i; + + for (i = 0; i < tp->nr_args; i++) + traceprobe_free_probe_arg(&tp->args[i]); + + kfree(call->class->system); + kfree(call->name); + kfree(call->print_fmt); +} + +int trace_probe_init(struct trace_probe *tp, const char *event, + const char *group) +{ + struct trace_event_call *call = trace_probe_event_call(tp); + + if (!event || !group) + return -EINVAL; + + call->class = &tp->class; + call->name = kstrdup(event, GFP_KERNEL); + if (!call->name) + return -ENOMEM; + + tp->class.system = kstrdup(group, GFP_KERNEL); + if (!tp->class.system) { + kfree(call->name); + call->name = NULL; + return -ENOMEM; + } + INIT_LIST_HEAD(&tp->files); + INIT_LIST_HEAD(&tp->class.fields); + + return 0; +} + +int trace_probe_register_event_call(struct trace_probe *tp) +{ + struct trace_event_call *call = trace_probe_event_call(tp); + int ret; + + ret = register_trace_event(&call->event); + if (!ret) + return -ENODEV; + + ret = trace_add_event_call(call); + if (ret) + unregister_trace_event(&call->event); + + return ret; +} + +int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file) +{ + struct event_file_link *link; + + link = kmalloc(sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; + + link->file = file; + INIT_LIST_HEAD(&link->list); + list_add_tail_rcu(&link->list, &tp->files); + trace_probe_set_flag(tp, TP_FLAG_TRACE); + return 0; +} + +struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp, + struct trace_event_file *file) +{ + struct event_file_link *link; + + trace_probe_for_each_link(link, tp) { + if (link->file == file) + return link; + } + + return NULL; +} + +int trace_probe_remove_file(struct trace_probe *tp, + struct trace_event_file *file) +{ + struct event_file_link *link; + + link = trace_probe_get_file_link(tp, file); + if (!link) + return -ENOENT; + + list_del_rcu(&link->list); + synchronize_rcu(); + kfree(link); + + if (list_empty(&tp->files)) + trace_probe_clear_flag(tp, TP_FLAG_TRACE); + + return 0; +} diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index f9a8c632188b..d1714820efe1 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -55,7 +55,6 @@ /* Flags for trace_probe */ #define TP_FLAG_TRACE 1 #define TP_FLAG_PROFILE 2 -#define TP_FLAG_REGISTERED 4 /* data_loc: data location, compatible with u32 */ #define make_data_loc(len, offs) \ @@ -92,10 +91,13 @@ enum fetch_op { FETCH_OP_FOFFS, /* File offset: .immediate */ // Stage 2 (dereference) op FETCH_OP_DEREF, /* Dereference: .offset */ + FETCH_OP_UDEREF, /* User-space Dereference: .offset */ // Stage 3 (store) ops FETCH_OP_ST_RAW, /* Raw: .size */ FETCH_OP_ST_MEM, /* Mem: .offset, .size */ + FETCH_OP_ST_UMEM, /* Mem: .offset, .size */ FETCH_OP_ST_STRING, /* String: .offset, .size */ + FETCH_OP_ST_USTRING, /* User String: .offset, .size */ // Stage 4 (modify) op FETCH_OP_MOD_BF, /* Bitfield: .basesize, .lshift, .rshift */ // Stage 5 (loop) op @@ -235,16 +237,71 @@ struct event_file_link { struct list_head list; }; +static inline bool trace_probe_test_flag(struct trace_probe *tp, + unsigned int flag) +{ + return !!(tp->flags & flag); +} + +static inline void trace_probe_set_flag(struct trace_probe *tp, + unsigned int flag) +{ + tp->flags |= flag; +} + +static inline void trace_probe_clear_flag(struct trace_probe *tp, + unsigned int flag) +{ + tp->flags &= ~flag; +} + static inline bool trace_probe_is_enabled(struct trace_probe *tp) { - return !!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE)); + return trace_probe_test_flag(tp, TP_FLAG_TRACE | TP_FLAG_PROFILE); } -static inline bool trace_probe_is_registered(struct trace_probe *tp) +static inline const char *trace_probe_name(struct trace_probe *tp) { - return !!(tp->flags & TP_FLAG_REGISTERED); + return trace_event_name(&tp->call); } +static inline const char *trace_probe_group_name(struct trace_probe *tp) +{ + return tp->call.class->system; +} + +static inline struct trace_event_call * + trace_probe_event_call(struct trace_probe *tp) +{ + return &tp->call; +} + +static inline int trace_probe_unregister_event_call(struct trace_probe *tp) +{ + /* tp->event is unregistered in trace_remove_event_call() */ + return trace_remove_event_call(&tp->call); +} + +static inline bool trace_probe_has_single_file(struct trace_probe *tp) +{ + return !!list_is_singular(&tp->files); +} + +int trace_probe_init(struct trace_probe *tp, const char *event, + const char *group); +void trace_probe_cleanup(struct trace_probe *tp); +int trace_probe_register_event_call(struct trace_probe *tp); +int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file); +int trace_probe_remove_file(struct trace_probe *tp, + struct trace_event_file *file); +struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp, + struct trace_event_file *file); + +#define trace_probe_for_each_link(pos, tp) \ + list_for_each_entry(pos, &(tp)->files, list) +#define trace_probe_for_each_link_rcu(pos, tp) \ + list_for_each_entry_rcu(pos, &(tp)->files, list) + /* Check the name is good for event/group/fields */ static inline bool is_good_name(const char *name) { @@ -257,18 +314,6 @@ static inline bool is_good_name(const char *name) return true; } -static inline struct event_file_link * -find_event_file_link(struct trace_probe *tp, struct trace_event_file *file) -{ - struct event_file_link *link; - - list_for_each_entry(link, &tp->files, list) - if (link->file == file) - return link; - - return NULL; -} - #define TPARG_FL_RETURN BIT(0) #define TPARG_FL_KERNEL BIT(1) #define TPARG_FL_FENTRY BIT(2) diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h index c30c61f12ddd..e5282828f4a6 100644 --- a/kernel/trace/trace_probe_tmpl.h +++ b/kernel/trace/trace_probe_tmpl.h @@ -59,8 +59,13 @@ process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, static nokprobe_inline int fetch_store_strlen(unsigned long addr); static nokprobe_inline int fetch_store_string(unsigned long addr, void *dest, void *base); +static nokprobe_inline int fetch_store_strlen_user(unsigned long addr); +static nokprobe_inline int +fetch_store_string_user(unsigned long addr, void *dest, void *base); static nokprobe_inline int probe_mem_read(void *dest, void *src, size_t size); +static nokprobe_inline int +probe_mem_read_user(void *dest, void *src, size_t size); /* From the 2nd stage, routine is same */ static nokprobe_inline int @@ -74,14 +79,21 @@ process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val, stage2: /* 2nd stage: dereference memory if needed */ - while (code->op == FETCH_OP_DEREF) { - lval = val; - ret = probe_mem_read(&val, (void *)val + code->offset, - sizeof(val)); + do { + if (code->op == FETCH_OP_DEREF) { + lval = val; + ret = probe_mem_read(&val, (void *)val + code->offset, + sizeof(val)); + } else if (code->op == FETCH_OP_UDEREF) { + lval = val; + ret = probe_mem_read_user(&val, + (void *)val + code->offset, sizeof(val)); + } else + break; if (ret) return ret; code++; - } + } while (1); s3 = code; stage3: @@ -91,6 +103,10 @@ stage3: ret = fetch_store_strlen(val + code->offset); code++; goto array; + } else if (code->op == FETCH_OP_ST_USTRING) { + ret += fetch_store_strlen_user(val + code->offset); + code++; + goto array; } else return -EILSEQ; } @@ -102,10 +118,17 @@ stage3: case FETCH_OP_ST_MEM: probe_mem_read(dest, (void *)val + code->offset, code->size); break; + case FETCH_OP_ST_UMEM: + probe_mem_read_user(dest, (void *)val + code->offset, code->size); + break; case FETCH_OP_ST_STRING: loc = *(u32 *)dest; ret = fetch_store_string(val + code->offset, dest, base); break; + case FETCH_OP_ST_USTRING: + loc = *(u32 *)dest; + ret = fetch_store_string_user(val + code->offset, dest, base); + break; default: return -EILSEQ; } @@ -123,7 +146,8 @@ array: total += ret; if (++i < code->param) { code = s3; - if (s3->op != FETCH_OP_ST_STRING) { + if (s3->op != FETCH_OP_ST_STRING && + s3->op != FETCH_OP_ST_USTRING) { dest += s3->size; val += s3->size; goto stage3; diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 7860e3f59fad..1ceedb9146b1 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -140,6 +140,13 @@ probe_mem_read(void *dest, void *src, size_t size) return copy_from_user(dest, vaddr, size) ? -EFAULT : 0; } + +static nokprobe_inline int +probe_mem_read_user(void *dest, void *src, size_t size) +{ + return probe_mem_read(dest, src, size); +} + /* * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max * length and relative data location. @@ -176,6 +183,12 @@ fetch_store_string(unsigned long addr, void *dest, void *base) return ret; } +static nokprobe_inline int +fetch_store_string_user(unsigned long addr, void *dest, void *base) +{ + return fetch_store_string(addr, dest, base); +} + /* Return the length of string -- including null terminal byte */ static nokprobe_inline int fetch_store_strlen(unsigned long addr) @@ -191,6 +204,12 @@ fetch_store_strlen(unsigned long addr) return (len > MAX_STRING_SIZE) ? 0 : len; } +static nokprobe_inline int +fetch_store_strlen_user(unsigned long addr) +{ + return fetch_store_strlen(addr); +} + static unsigned long translate_user_vaddr(unsigned long file_offset) { unsigned long base_addr; @@ -270,8 +289,8 @@ static bool trace_uprobe_match(const char *system, const char *event, { struct trace_uprobe *tu = to_trace_uprobe(ev); - return strcmp(trace_event_name(&tu->tp.call), event) == 0 && - (!system || strcmp(tu->tp.call.class->system, system) == 0); + return strcmp(trace_probe_name(&tu->tp), event) == 0 && + (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0); } /* @@ -281,25 +300,17 @@ static struct trace_uprobe * alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) { struct trace_uprobe *tu; - - if (!event || !group) - return ERR_PTR(-EINVAL); + int ret; tu = kzalloc(SIZEOF_TRACE_UPROBE(nargs), GFP_KERNEL); if (!tu) return ERR_PTR(-ENOMEM); - tu->tp.call.class = &tu->tp.class; - tu->tp.call.name = kstrdup(event, GFP_KERNEL); - if (!tu->tp.call.name) - goto error; - - tu->tp.class.system = kstrdup(group, GFP_KERNEL); - if (!tu->tp.class.system) + ret = trace_probe_init(&tu->tp, event, group); + if (ret < 0) goto error; dyn_event_init(&tu->devent, &trace_uprobe_ops); - INIT_LIST_HEAD(&tu->tp.files); tu->consumer.handler = uprobe_dispatcher; if (is_ret) tu->consumer.ret_handler = uretprobe_dispatcher; @@ -307,25 +318,18 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) return tu; error: - kfree(tu->tp.call.name); kfree(tu); - return ERR_PTR(-ENOMEM); + return ERR_PTR(ret); } static void free_trace_uprobe(struct trace_uprobe *tu) { - int i; - if (!tu) return; - for (i = 0; i < tu->tp.nr_args; i++) - traceprobe_free_probe_arg(&tu->tp.args[i]); - path_put(&tu->path); - kfree(tu->tp.call.class->system); - kfree(tu->tp.call.name); + trace_probe_cleanup(&tu->tp); kfree(tu->filename); kfree(tu); } @@ -336,8 +340,8 @@ static struct trace_uprobe *find_probe_event(const char *event, const char *grou struct trace_uprobe *tu; for_each_trace_uprobe(tu, pos) - if (strcmp(trace_event_name(&tu->tp.call), event) == 0 && - strcmp(tu->tp.call.class->system, group) == 0) + if (strcmp(trace_probe_name(&tu->tp), event) == 0 && + strcmp(trace_probe_group_name(&tu->tp), group) == 0) return tu; return NULL; @@ -372,8 +376,8 @@ static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new) struct trace_uprobe *tmp, *old = NULL; struct inode *new_inode = d_real_inode(new->path.dentry); - old = find_probe_event(trace_event_name(&new->tp.call), - new->tp.call.class->system); + old = find_probe_event(trace_probe_name(&new->tp), + trace_probe_group_name(&new->tp)); for_each_trace_uprobe(tmp, pos) { if ((old ? old != tmp : true) && @@ -578,6 +582,10 @@ static int trace_uprobe_create(int argc, const char **argv) goto error; } + ret = traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)); + if (ret < 0) + goto error; + ret = register_trace_uprobe(tu); if (!ret) goto out; @@ -621,8 +629,8 @@ static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev) char c = is_ret_probe(tu) ? 'r' : 'p'; int i; - seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, tu->tp.call.class->system, - trace_event_name(&tu->tp.call), tu->filename, + seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp), + trace_probe_name(&tu->tp), tu->filename, (int)(sizeof(void *) * 2), tu->offset); if (tu->ref_ctr_offset) @@ -692,7 +700,7 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) tu = to_trace_uprobe(ev); seq_printf(m, " %s %-44s %15lu\n", tu->filename, - trace_event_name(&tu->tp.call), tu->nhit); + trace_probe_name(&tu->tp), tu->nhit); return 0; } @@ -818,7 +826,7 @@ static void __uprobe_trace_func(struct trace_uprobe *tu, struct ring_buffer *buffer; void *data; int size, esize; - struct trace_event_call *call = &tu->tp.call; + struct trace_event_call *call = trace_probe_event_call(&tu->tp); WARN_ON(call != trace_file->event_call); @@ -860,7 +868,7 @@ static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, return 0; rcu_read_lock(); - list_for_each_entry_rcu(link, &tu->tp.files, list) + trace_probe_for_each_link_rcu(link, &tu->tp) __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file); rcu_read_unlock(); @@ -874,7 +882,7 @@ static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func, struct event_file_link *link; rcu_read_lock(); - list_for_each_entry_rcu(link, &tu->tp.files, list) + trace_probe_for_each_link_rcu(link, &tu->tp) __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file); rcu_read_unlock(); } @@ -893,12 +901,12 @@ print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *e if (is_ret_probe(tu)) { trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", - trace_event_name(&tu->tp.call), + trace_probe_name(&tu->tp), entry->vaddr[1], entry->vaddr[0]); data = DATAOF_TRACE_ENTRY(entry, true); } else { trace_seq_printf(s, "%s: (0x%lx)", - trace_event_name(&tu->tp.call), + trace_probe_name(&tu->tp), entry->vaddr[0]); data = DATAOF_TRACE_ENTRY(entry, false); } @@ -921,26 +929,20 @@ probe_event_enable(struct trace_uprobe *tu, struct trace_event_file *file, filter_func_t filter) { bool enabled = trace_probe_is_enabled(&tu->tp); - struct event_file_link *link = NULL; int ret; if (file) { - if (tu->tp.flags & TP_FLAG_PROFILE) + if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) return -EINTR; - link = kmalloc(sizeof(*link), GFP_KERNEL); - if (!link) - return -ENOMEM; - - link->file = file; - list_add_tail_rcu(&link->list, &tu->tp.files); - - tu->tp.flags |= TP_FLAG_TRACE; + ret = trace_probe_add_file(&tu->tp, file); + if (ret < 0) + return ret; } else { - if (tu->tp.flags & TP_FLAG_TRACE) + if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) return -EINTR; - tu->tp.flags |= TP_FLAG_PROFILE; + trace_probe_set_flag(&tu->tp, TP_FLAG_PROFILE); } WARN_ON(!uprobe_filter_is_empty(&tu->filter)); @@ -970,13 +972,11 @@ probe_event_enable(struct trace_uprobe *tu, struct trace_event_file *file, uprobe_buffer_disable(); err_flags: - if (file) { - list_del(&link->list); - kfree(link); - tu->tp.flags &= ~TP_FLAG_TRACE; - } else { - tu->tp.flags &= ~TP_FLAG_PROFILE; - } + if (file) + trace_probe_remove_file(&tu->tp, file); + else + trace_probe_clear_flag(&tu->tp, TP_FLAG_PROFILE); + return ret; } @@ -987,26 +987,18 @@ probe_event_disable(struct trace_uprobe *tu, struct trace_event_file *file) return; if (file) { - struct event_file_link *link; - - link = find_event_file_link(&tu->tp, file); - if (!link) + if (trace_probe_remove_file(&tu->tp, file) < 0) return; - list_del_rcu(&link->list); - /* synchronize with u{,ret}probe_trace_func */ - synchronize_rcu(); - kfree(link); - - if (!list_empty(&tu->tp.files)) + if (trace_probe_is_enabled(&tu->tp)) return; - } + } else + trace_probe_clear_flag(&tu->tp, TP_FLAG_PROFILE); WARN_ON(!uprobe_filter_is_empty(&tu->filter)); uprobe_unregister(tu->inode, tu->offset, &tu->consumer); tu->inode = NULL; - tu->tp.flags &= file ? ~TP_FLAG_TRACE : ~TP_FLAG_PROFILE; uprobe_buffer_disable(); } @@ -1126,7 +1118,7 @@ static void __uprobe_perf_func(struct trace_uprobe *tu, unsigned long func, struct pt_regs *regs, struct uprobe_cpu_buffer *ucb, int dsize) { - struct trace_event_call *call = &tu->tp.call; + struct trace_event_call *call = trace_probe_event_call(&tu->tp); struct uprobe_trace_entry_head *entry; struct hlist_head *head; void *data; @@ -1279,11 +1271,11 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) ucb = uprobe_buffer_get(); store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize); - if (tu->tp.flags & TP_FLAG_TRACE) + if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) ret |= uprobe_trace_func(tu, regs, ucb, dsize); #ifdef CONFIG_PERF_EVENTS - if (tu->tp.flags & TP_FLAG_PROFILE) + if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) ret |= uprobe_perf_func(tu, regs, ucb, dsize); #endif uprobe_buffer_put(ucb); @@ -1314,11 +1306,11 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con, ucb = uprobe_buffer_get(); store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize); - if (tu->tp.flags & TP_FLAG_TRACE) + if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE)) uretprobe_trace_func(tu, func, regs, ucb, dsize); #ifdef CONFIG_PERF_EVENTS - if (tu->tp.flags & TP_FLAG_PROFILE) + if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE)) uretprobe_perf_func(tu, func, regs, ucb, dsize); #endif uprobe_buffer_put(ucb); @@ -1329,10 +1321,10 @@ static struct trace_event_functions uprobe_funcs = { .trace = print_uprobe_event }; -static inline void init_trace_event_call(struct trace_uprobe *tu, - struct trace_event_call *call) +static inline void init_trace_event_call(struct trace_uprobe *tu) { - INIT_LIST_HEAD(&call->class->fields); + struct trace_event_call *call = trace_probe_event_call(&tu->tp); + call->event.funcs = &uprobe_funcs; call->class->define_fields = uprobe_event_define_fields; @@ -1343,43 +1335,14 @@ static inline void init_trace_event_call(struct trace_uprobe *tu, static int register_uprobe_event(struct trace_uprobe *tu) { - struct trace_event_call *call = &tu->tp.call; - int ret = 0; - - init_trace_event_call(tu, call); - - if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) - return -ENOMEM; + init_trace_event_call(tu); - ret = register_trace_event(&call->event); - if (!ret) { - kfree(call->print_fmt); - return -ENODEV; - } - - ret = trace_add_event_call(call); - - if (ret) { - pr_info("Failed to register uprobe event: %s\n", - trace_event_name(call)); - kfree(call->print_fmt); - unregister_trace_event(&call->event); - } - - return ret; + return trace_probe_register_event_call(&tu->tp); } static int unregister_uprobe_event(struct trace_uprobe *tu) { - int ret; - - /* tu->event is unregistered in trace_remove_event_call() */ - ret = trace_remove_event_call(&tu->tp.call); - if (ret) - return ret; - kfree(tu->tp.call.print_fmt); - tu->tp.call.print_fmt = NULL; - return 0; + return trace_probe_unregister_event_call(&tu->tp); } #ifdef CONFIG_PERF_EVENTS @@ -1419,14 +1382,14 @@ create_local_trace_uprobe(char *name, unsigned long offs, tu->path = path; tu->ref_ctr_offset = ref_ctr_offset; tu->filename = kstrdup(name, GFP_KERNEL); - init_trace_event_call(tu, &tu->tp.call); + init_trace_event_call(tu); if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) { ret = -ENOMEM; goto error; } - return &tu->tp.call; + return trace_probe_event_call(&tu->tp); error: free_trace_uprobe(tu); return ERR_PTR(ret); @@ -1438,9 +1401,6 @@ void destroy_local_trace_uprobe(struct trace_event_call *event_call) tu = container_of(event_call, struct trace_uprobe, tp.call); - kfree(tu->tp.call.print_fmt); - tu->tp.call.print_fmt = NULL; - free_trace_uprobe(tu); } #endif /* CONFIG_PERF_EVENTS */ diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index df3ade14ccbd..73956eaff8a9 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c @@ -55,8 +55,8 @@ struct tp_probes { static inline void *allocate_probes(int count) { - struct tp_probes *p = kmalloc(count * sizeof(struct tracepoint_func) - + sizeof(struct tp_probes), GFP_KERNEL); + struct tp_probes *p = kmalloc(struct_size(p, probes, count), + GFP_KERNEL); return p == NULL ? NULL : p->probes; } diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a858b55e8ac7..bc6673ab3a08 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2076,6 +2076,14 @@ config TEST_STACKINIT If unsure, say N. +config TEST_MEMINIT + tristate "Test heap/page initialization" + help + Test if the kernel is zero-initializing heap and page allocations. + This can be useful to test init_on_alloc and init_on_free features. + + If unsure, say N. + endif # RUNTIME_TESTING_MENU config MEMTEST diff --git a/lib/Makefile b/lib/Makefile index fdd56bc219b8..59067f51f3ab 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -92,6 +92,7 @@ obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o obj-$(CONFIG_TEST_STACKINIT) += test_stackinit.o obj-$(CONFIG_TEST_BLACKHOLE_DEV) += test_blackhole_dev.o +obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o obj-$(CONFIG_TEST_LIVEPATCH) += livepatch/ diff --git a/lib/ioremap.c b/lib/ioremap.c index 063213685563..0a2ffadc6d71 100644 --- a/lib/ioremap.c +++ b/lib/ioremap.c @@ -30,6 +30,8 @@ early_param("nohugeiomap", set_nohugeiomap); void __init ioremap_huge_init(void) { if (!ioremap_huge_disabled) { + if (arch_ioremap_p4d_supported()) + ioremap_p4d_capable = 1; if (arch_ioremap_pud_supported()) ioremap_pud_capable = 1; if (arch_ioremap_pmd_supported()) @@ -86,6 +88,9 @@ static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, if ((end - addr) != PMD_SIZE) return 0; + if (!IS_ALIGNED(addr, PMD_SIZE)) + return 0; + if (!IS_ALIGNED(phys_addr, PMD_SIZE)) return 0; @@ -126,6 +131,9 @@ static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, if ((end - addr) != PUD_SIZE) return 0; + if (!IS_ALIGNED(addr, PUD_SIZE)) + return 0; + if (!IS_ALIGNED(phys_addr, PUD_SIZE)) return 0; @@ -166,6 +174,9 @@ static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, if ((end - addr) != P4D_SIZE) return 0; + if (!IS_ALIGNED(addr, P4D_SIZE)) + return 0; + if (!IS_ALIGNED(phys_addr, P4D_SIZE)) return 0; diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h index 08c60d10747f..3bb6260d8f42 100644 --- a/lib/mpi/longlong.h +++ b/lib/mpi/longlong.h @@ -397,8 +397,8 @@ do { \ #define add_ssaaaa(sh, sl, ah, al, bh, bl) \ __asm__ ("addl %5,%1\n" \ "adcl %3,%0" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "%0" ((USItype)(ah)), \ "g" ((USItype)(bh)), \ "%1" ((USItype)(al)), \ @@ -406,22 +406,22 @@ do { \ #define sub_ddmmss(sh, sl, ah, al, bh, bl) \ __asm__ ("subl %5,%1\n" \ "sbbl %3,%0" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "0" ((USItype)(ah)), \ "g" ((USItype)(bh)), \ "1" ((USItype)(al)), \ "g" ((USItype)(bl))) #define umul_ppmm(w1, w0, u, v) \ __asm__ ("mull %3" \ - : "=a" ((USItype)(w0)), \ - "=d" ((USItype)(w1)) \ + : "=a" (w0), \ + "=d" (w1) \ : "%0" ((USItype)(u)), \ "rm" ((USItype)(v))) #define udiv_qrnnd(q, r, n1, n0, d) \ __asm__ ("divl %4" \ - : "=a" ((USItype)(q)), \ - "=d" ((USItype)(r)) \ + : "=a" (q), \ + "=d" (r) \ : "0" ((USItype)(n0)), \ "1" ((USItype)(n1)), \ "rm" ((USItype)(d))) diff --git a/lib/rbtree.c b/lib/rbtree.c index 1ef6e25d031c..abc86c6a3177 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -83,14 +83,10 @@ __rb_rotate_set_parents(struct rb_node *old, struct rb_node *new, static __always_inline void __rb_insert(struct rb_node *node, struct rb_root *root, - bool newleft, struct rb_node **leftmost, void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) { struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; - if (newleft) - *leftmost = node; - while (true) { /* * Loop invariant: node is red. @@ -437,38 +433,19 @@ static const struct rb_augment_callbacks dummy_callbacks = { void rb_insert_color(struct rb_node *node, struct rb_root *root) { - __rb_insert(node, root, false, NULL, dummy_rotate); + __rb_insert(node, root, dummy_rotate); } EXPORT_SYMBOL(rb_insert_color); void rb_erase(struct rb_node *node, struct rb_root *root) { struct rb_node *rebalance; - rebalance = __rb_erase_augmented(node, root, - NULL, &dummy_callbacks); + rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); if (rebalance) ____rb_erase_color(rebalance, root, dummy_rotate); } EXPORT_SYMBOL(rb_erase); -void rb_insert_color_cached(struct rb_node *node, - struct rb_root_cached *root, bool leftmost) -{ - __rb_insert(node, &root->rb_root, leftmost, - &root->rb_leftmost, dummy_rotate); -} -EXPORT_SYMBOL(rb_insert_color_cached); - -void rb_erase_cached(struct rb_node *node, struct rb_root_cached *root) -{ - struct rb_node *rebalance; - rebalance = __rb_erase_augmented(node, &root->rb_root, - &root->rb_leftmost, &dummy_callbacks); - if (rebalance) - ____rb_erase_color(rebalance, &root->rb_root, dummy_rotate); -} -EXPORT_SYMBOL(rb_erase_cached); - /* * Augmented rbtree manipulation functions. * @@ -477,10 +454,9 @@ EXPORT_SYMBOL(rb_erase_cached); */ void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, - bool newleft, struct rb_node **leftmost, void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) { - __rb_insert(node, root, newleft, leftmost, augment_rotate); + __rb_insert(node, root, augment_rotate); } EXPORT_SYMBOL(__rb_insert_augmented); @@ -591,16 +567,6 @@ void rb_replace_node(struct rb_node *victim, struct rb_node *new, } EXPORT_SYMBOL(rb_replace_node); -void rb_replace_node_cached(struct rb_node *victim, struct rb_node *new, - struct rb_root_cached *root) -{ - rb_replace_node(victim, new, &root->rb_root); - - if (root->rb_leftmost == victim) - root->rb_leftmost = new; -} -EXPORT_SYMBOL(rb_replace_node_cached); - void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new, struct rb_root *root) { diff --git a/lib/string.c b/lib/string.c index 6016eb3ac73d..461fb620f85f 100644 --- a/lib/string.c +++ b/lib/string.c @@ -400,6 +400,9 @@ EXPORT_SYMBOL(strncmp); * strchr - Find the first occurrence of a character in a string * @s: The string to be searched * @c: The character to search for + * + * Note that the %NUL-terminator is considered part of the string, and can + * be searched for. */ char *strchr(const char *s, int c) { @@ -453,12 +456,18 @@ EXPORT_SYMBOL(strrchr); * @s: The string to be searched * @count: The number of characters to be searched * @c: The character to search for + * + * Note that the %NUL-terminator is considered part of the string, and can + * be searched for. */ char *strnchr(const char *s, size_t count, int c) { - for (; count-- && *s != '\0'; ++s) + while (count--) { if (*s == (char)c) return (char *)s; + if (*s++ == '\0') + break; + } return NULL; } EXPORT_SYMBOL(strnchr); diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 3a90a9e2b94a..963050c0283e 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -231,35 +231,36 @@ static bool unescape_special(char **src, char **dst) * @src: source buffer (escaped) * @dst: destination buffer (unescaped) * @size: size of the destination buffer (0 to unlimit) - * @flags: combination of the flags (bitwise OR): - * %UNESCAPE_SPACE: + * @flags: combination of the flags. + * + * Description: + * The function unquotes characters in the given string. + * + * Because the size of the output will be the same as or less than the size of + * the input, the transformation may be performed in place. + * + * Caller must provide valid source and destination pointers. Be aware that + * destination buffer will always be NULL-terminated. Source string must be + * NULL-terminated as well. The supported flags are:: + * + * UNESCAPE_SPACE: * '\f' - form feed * '\n' - new line * '\r' - carriage return * '\t' - horizontal tab * '\v' - vertical tab - * %UNESCAPE_OCTAL: + * UNESCAPE_OCTAL: * '\NNN' - byte with octal value NNN (1 to 3 digits) - * %UNESCAPE_HEX: + * UNESCAPE_HEX: * '\xHH' - byte with hexadecimal value HH (1 to 2 digits) - * %UNESCAPE_SPECIAL: + * UNESCAPE_SPECIAL: * '\"' - double quote * '\\' - backslash * '\a' - alert (BEL) * '\e' - escape - * %UNESCAPE_ANY: + * UNESCAPE_ANY: * all previous together * - * Description: - * The function unquotes characters in the given string. - * - * Because the size of the output will be the same as or less than the size of - * the input, the transformation may be performed in place. - * - * Caller must provide valid source and destination pointers. Be aware that - * destination buffer will always be NULL-terminated. Source string must be - * NULL-terminated as well. - * * Return: * The amount of the characters processed to the destination buffer excluding * trailing '\0' is returned. @@ -441,7 +442,29 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * @isz: source buffer size * @dst: destination buffer (escaped) * @osz: destination buffer size - * @flags: combination of the flags (bitwise OR): + * @flags: combination of the flags + * @only: NULL-terminated string containing characters used to limit + * the selected escape class. If characters are included in @only + * that would not normally be escaped by the classes selected + * in @flags, they will be copied to @dst unescaped. + * + * Description: + * The process of escaping byte buffer includes several parts. They are applied + * in the following sequence. + * + * 1. The character is matched to the printable class, if asked, and in + * case of match it passes through to the output. + * 2. The character is not matched to the one from @only string and thus + * must go as-is to the output. + * 3. The character is checked if it falls into the class given by @flags. + * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any + * character. Note that they actually can't go together, otherwise + * %ESCAPE_HEX will be ignored. + * + * Caller must provide valid source and destination pointers. Be aware that + * destination buffer will not be NULL-terminated, thus caller have to append + * it if needs. The supported flags are:: + * * %ESCAPE_SPACE: (special white space, not space itself) * '\f' - form feed * '\n' - new line @@ -464,26 +487,6 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * all previous together * %ESCAPE_HEX: * '\xHH' - byte with hexadecimal value HH (2 digits) - * @only: NULL-terminated string containing characters used to limit - * the selected escape class. If characters are included in @only - * that would not normally be escaped by the classes selected - * in @flags, they will be copied to @dst unescaped. - * - * Description: - * The process of escaping byte buffer includes several parts. They are applied - * in the following sequence. - * 1. The character is matched to the printable class, if asked, and in - * case of match it passes through to the output. - * 2. The character is not matched to the one from @only string and thus - * must go as-is to the output. - * 3. The character is checked if it falls into the class given by @flags. - * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any - * character. Note that they actually can't go together, otherwise - * %ESCAPE_HEX will be ignored. - * - * Caller must provide valid source and destination pointers. Be aware that - * destination buffer will not be NULL-terminated, thus caller have to append - * it if needs. * * Return: * The total size of the escaped output that would be generated for diff --git a/lib/test_meminit.c b/lib/test_meminit.c new file mode 100644 index 000000000000..62d19f270cad --- /dev/null +++ b/lib/test_meminit.c @@ -0,0 +1,364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for SL[AOU]B/page initialization at alloc/free time. + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/vmalloc.h> + +#define GARBAGE_INT (0x09A7BA9E) +#define GARBAGE_BYTE (0x9E) + +#define REPORT_FAILURES_IN_FN() \ + do { \ + if (failures) \ + pr_info("%s failed %d out of %d times\n", \ + __func__, failures, num_tests); \ + else \ + pr_info("all %d tests in %s passed\n", \ + num_tests, __func__); \ + } while (0) + +/* Calculate the number of uninitialized bytes in the buffer. */ +static int __init count_nonzero_bytes(void *ptr, size_t size) +{ + int i, ret = 0; + unsigned char *p = (unsigned char *)ptr; + + for (i = 0; i < size; i++) + if (p[i]) + ret++; + return ret; +} + +/* Fill a buffer with garbage, skipping |skip| first bytes. */ +static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) +{ + unsigned int *p = (unsigned int *)((char *)ptr + skip); + int i = 0; + + WARN_ON(skip > size); + size -= skip; + + while (size >= sizeof(*p)) { + p[i] = GARBAGE_INT; + i++; + size -= sizeof(*p); + } + if (size) + memset(&p[i], GARBAGE_BYTE, size); +} + +static void __init fill_with_garbage(void *ptr, size_t size) +{ + fill_with_garbage_skip(ptr, size, 0); +} + +static int __init do_alloc_pages_order(int order, int *total_failures) +{ + struct page *page; + void *buf; + size_t size = PAGE_SIZE << order; + + page = alloc_pages(GFP_KERNEL, order); + buf = page_address(page); + fill_with_garbage(buf, size); + __free_pages(page, order); + + page = alloc_pages(GFP_KERNEL, order); + buf = page_address(page); + if (count_nonzero_bytes(buf, size)) + (*total_failures)++; + fill_with_garbage(buf, size); + __free_pages(page, order); + return 1; +} + +/* Test the page allocator by calling alloc_pages with different orders. */ +static int __init test_pages(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i; + + for (i = 0; i < 10; i++) + num_tests += do_alloc_pages_order(i, &failures); + + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +/* Test kmalloc() with given parameters. */ +static int __init do_kmalloc_size(size_t size, int *total_failures) +{ + void *buf; + + buf = kmalloc(size, GFP_KERNEL); + fill_with_garbage(buf, size); + kfree(buf); + + buf = kmalloc(size, GFP_KERNEL); + if (count_nonzero_bytes(buf, size)) + (*total_failures)++; + fill_with_garbage(buf, size); + kfree(buf); + return 1; +} + +/* Test vmalloc() with given parameters. */ +static int __init do_vmalloc_size(size_t size, int *total_failures) +{ + void *buf; + + buf = vmalloc(size); + fill_with_garbage(buf, size); + vfree(buf); + + buf = vmalloc(size); + if (count_nonzero_bytes(buf, size)) + (*total_failures)++; + fill_with_garbage(buf, size); + vfree(buf); + return 1; +} + +/* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ +static int __init test_kvmalloc(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i, size; + + for (i = 0; i < 20; i++) { + size = 1 << i; + num_tests += do_kmalloc_size(size, &failures); + num_tests += do_vmalloc_size(size, &failures); + } + + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +#define CTOR_BYTES (sizeof(unsigned int)) +#define CTOR_PATTERN (0x41414141) +/* Initialize the first 4 bytes of the object. */ +static void test_ctor(void *obj) +{ + *(unsigned int *)obj = CTOR_PATTERN; +} + +/* + * Check the invariants for the buffer allocated from a slab cache. + * If the cache has a test constructor, the first 4 bytes of the object must + * always remain equal to CTOR_PATTERN. + * If the cache isn't an RCU-typesafe one, or if the allocation is done with + * __GFP_ZERO, then the object contents must be zeroed after allocation. + * If the cache is an RCU-typesafe one, the object contents must never be + * zeroed after the first use. This is checked by memcmp() in + * do_kmem_cache_size(). + */ +static bool __init check_buf(void *buf, int size, bool want_ctor, + bool want_rcu, bool want_zero) +{ + int bytes; + bool fail = false; + + bytes = count_nonzero_bytes(buf, size); + WARN_ON(want_ctor && want_zero); + if (want_zero) + return bytes; + if (want_ctor) { + if (*(unsigned int *)buf != CTOR_PATTERN) + fail = 1; + } else { + if (bytes) + fail = !want_rcu; + } + return fail; +} + +/* + * Test kmem_cache with given parameters: + * want_ctor - use a constructor; + * want_rcu - use SLAB_TYPESAFE_BY_RCU; + * want_zero - use __GFP_ZERO. + */ +static int __init do_kmem_cache_size(size_t size, bool want_ctor, + bool want_rcu, bool want_zero, + int *total_failures) +{ + struct kmem_cache *c; + int iter; + bool fail = false; + gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); + void *buf, *buf_copy; + + c = kmem_cache_create("test_cache", size, 1, + want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, + want_ctor ? test_ctor : NULL); + for (iter = 0; iter < 10; iter++) { + buf = kmem_cache_alloc(c, alloc_mask); + /* Check that buf is zeroed, if it must be. */ + fail = check_buf(buf, size, want_ctor, want_rcu, want_zero); + fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); + + if (!want_rcu) { + kmem_cache_free(c, buf); + continue; + } + + /* + * If this is an RCU cache, use a critical section to ensure we + * can touch objects after they're freed. + */ + rcu_read_lock(); + /* + * Copy the buffer to check that it's not wiped on + * free(). + */ + buf_copy = kmalloc(size, GFP_KERNEL); + if (buf_copy) + memcpy(buf_copy, buf, size); + + kmem_cache_free(c, buf); + /* + * Check that |buf| is intact after kmem_cache_free(). + * |want_zero| is false, because we wrote garbage to + * the buffer already. + */ + fail |= check_buf(buf, size, want_ctor, want_rcu, + false); + if (buf_copy) { + fail |= (bool)memcmp(buf, buf_copy, size); + kfree(buf_copy); + } + rcu_read_unlock(); + } + kmem_cache_destroy(c); + + *total_failures += fail; + return 1; +} + +/* + * Check that the data written to an RCU-allocated object survives + * reallocation. + */ +static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) +{ + struct kmem_cache *c; + void *buf, *buf_contents, *saved_ptr; + void **used_objects; + int i, iter, maxiter = 1024; + bool fail = false; + + c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, + NULL); + buf = kmem_cache_alloc(c, GFP_KERNEL); + saved_ptr = buf; + fill_with_garbage(buf, size); + buf_contents = kmalloc(size, GFP_KERNEL); + if (!buf_contents) + goto out; + used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); + if (!used_objects) { + kfree(buf_contents); + goto out; + } + memcpy(buf_contents, buf, size); + kmem_cache_free(c, buf); + /* + * Run for a fixed number of iterations. If we never hit saved_ptr, + * assume the test passes. + */ + for (iter = 0; iter < maxiter; iter++) { + buf = kmem_cache_alloc(c, GFP_KERNEL); + used_objects[iter] = buf; + if (buf == saved_ptr) { + fail = memcmp(buf_contents, buf, size); + for (i = 0; i <= iter; i++) + kmem_cache_free(c, used_objects[i]); + goto free_out; + } + } + +free_out: + kmem_cache_destroy(c); + kfree(buf_contents); + kfree(used_objects); +out: + *total_failures += fail; + return 1; +} + +/* + * Test kmem_cache allocation by creating caches of different sizes, with and + * without constructors, with and without SLAB_TYPESAFE_BY_RCU. + */ +static int __init test_kmemcache(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i, flags, size; + bool ctor, rcu, zero; + + for (i = 0; i < 10; i++) { + size = 8 << i; + for (flags = 0; flags < 8; flags++) { + ctor = flags & 1; + rcu = flags & 2; + zero = flags & 4; + if (ctor & zero) + continue; + num_tests += do_kmem_cache_size(size, ctor, rcu, zero, + &failures); + } + } + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +/* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ +static int __init test_rcu_persistent(int *total_failures) +{ + int failures = 0, num_tests = 0; + int i, size; + + for (i = 0; i < 10; i++) { + size = 8 << i; + num_tests += do_kmem_cache_rcu_persistent(size, &failures); + } + REPORT_FAILURES_IN_FN(); + *total_failures += failures; + return num_tests; +} + +/* + * Run the tests. Each test function returns the number of executed tests and + * updates |failures| with the number of failed tests. + */ +static int __init test_meminit_init(void) +{ + int failures = 0, num_tests = 0; + + num_tests += test_pages(&failures); + num_tests += test_kvmalloc(&failures); + num_tests += test_kmemcache(&failures); + num_tests += test_rcu_persistent(&failures); + + if (failures == 0) + pr_info("all %d tests passed!\n", num_tests); + else + pr_info("failures: %d out of %d\n", failures, num_tests); + + return failures ? -EINVAL : 0; +} +module_init(test_meminit_init); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_overflow.c b/lib/test_overflow.c index fc680562d8b6..7a4b6f6c5473 100644 --- a/lib/test_overflow.c +++ b/lib/test_overflow.c @@ -486,16 +486,17 @@ static int __init test_overflow_shift(void) * Deal with the various forms of allocator arguments. See comments above * the DEFINE_TEST_ALLOC() instances for mapping of the "bits". */ -#define alloc010(alloc, arg, sz) alloc(sz, GFP_KERNEL) -#define alloc011(alloc, arg, sz) alloc(sz, GFP_KERNEL, NUMA_NO_NODE) +#define alloc_GFP (GFP_KERNEL | __GFP_NOWARN) +#define alloc010(alloc, arg, sz) alloc(sz, alloc_GFP) +#define alloc011(alloc, arg, sz) alloc(sz, alloc_GFP, NUMA_NO_NODE) #define alloc000(alloc, arg, sz) alloc(sz) #define alloc001(alloc, arg, sz) alloc(sz, NUMA_NO_NODE) -#define alloc110(alloc, arg, sz) alloc(arg, sz, GFP_KERNEL) +#define alloc110(alloc, arg, sz) alloc(arg, sz, alloc_GFP) #define free0(free, arg, ptr) free(ptr) #define free1(free, arg, ptr) free(arg, ptr) -/* Wrap around to 8K */ -#define TEST_SIZE (9 << PAGE_SHIFT) +/* Wrap around to 16K */ +#define TEST_SIZE (5 * 4096) #define DEFINE_TEST_ALLOC(func, free_func, want_arg, want_gfp, want_node)\ static int __init test_ ## func (void *arg) \ diff --git a/lib/test_string.c b/lib/test_string.c index bf8def01ed20..7b31f4a505bf 100644 --- a/lib/test_string.c +++ b/lib/test_string.c @@ -36,7 +36,7 @@ static __init int memset16_selftest(void) fail: kfree(p); if (i < 256) - return (i << 24) | (j << 16) | k; + return (i << 24) | (j << 16) | k | 0x8000; return 0; } @@ -72,7 +72,7 @@ static __init int memset32_selftest(void) fail: kfree(p); if (i < 256) - return (i << 24) | (j << 16) | k; + return (i << 24) | (j << 16) | k | 0x8000; return 0; } @@ -108,7 +108,74 @@ static __init int memset64_selftest(void) fail: kfree(p); if (i < 256) - return (i << 24) | (j << 16) | k; + return (i << 24) | (j << 16) | k | 0x8000; + return 0; +} + +static __init int strchr_selftest(void) +{ + const char *test_string = "abcdefghijkl"; + const char *empty_string = ""; + char *result; + int i; + + for (i = 0; i < strlen(test_string) + 1; i++) { + result = strchr(test_string, test_string[i]); + if (result - test_string != i) + return i + 'a'; + } + + result = strchr(empty_string, '\0'); + if (result != empty_string) + return 0x101; + + result = strchr(empty_string, 'a'); + if (result) + return 0x102; + + result = strchr(test_string, 'z'); + if (result) + return 0x103; + + return 0; +} + +static __init int strnchr_selftest(void) +{ + const char *test_string = "abcdefghijkl"; + const char *empty_string = ""; + char *result; + int i, j; + + for (i = 0; i < strlen(test_string) + 1; i++) { + for (j = 0; j < strlen(test_string) + 2; j++) { + result = strnchr(test_string, j, test_string[i]); + if (j <= i) { + if (!result) + continue; + return ((i + 'a') << 8) | j; + } + if (result - test_string != i) + return ((i + 'a') << 8) | j; + } + } + + result = strnchr(empty_string, 0, '\0'); + if (result) + return 0x10001; + + result = strnchr(empty_string, 1, '\0'); + if (result != empty_string) + return 0x10002; + + result = strnchr(empty_string, 1, 'a'); + if (result) + return 0x10003; + + result = strnchr(NULL, 0, '\0'); + if (result) + return 0x10004; + return 0; } @@ -131,6 +198,16 @@ static __init int string_selftest_init(void) if (subtest) goto fail; + test = 4; + subtest = strchr_selftest(); + if (subtest) + goto fail; + + test = 5; + subtest = strnchr_selftest(); + if (subtest) + goto fail; + pr_info("String selftests succeeded\n"); return 0; fail: diff --git a/mm/Kconfig b/mm/Kconfig index 495d7368ced8..56cec636a1fc 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -649,8 +649,7 @@ config IDLE_PAGE_TRACKING See Documentation/admin-guide/mm/idle_page_tracking.rst for more details. -# arch_add_memory() comprehends device memory -config ARCH_HAS_ZONE_DEVICE +config ARCH_HAS_PTE_DEVMAP bool config ZONE_DEVICE @@ -658,7 +657,7 @@ config ZONE_DEVICE depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE depends on SPARSEMEM_VMEMMAP - depends on ARCH_HAS_ZONE_DEVICE + depends on ARCH_HAS_PTE_DEVMAP select XARRAY_MULTI help @@ -278,6 +278,12 @@ int __init cma_declare_contiguous(phys_addr_t base, */ alignment = max(alignment, (phys_addr_t)PAGE_SIZE << max_t(unsigned long, MAX_ORDER - 1, pageblock_order)); + if (fixed && base & (alignment - 1)) { + ret = -EINVAL; + pr_err("Region at %pa must be aligned to %pa bytes\n", + &base, &alignment); + goto err; + } base = ALIGN(base, alignment); size = ALIGN(size, alignment); limit &= ~(alignment - 1); @@ -308,6 +314,13 @@ int __init cma_declare_contiguous(phys_addr_t base, if (limit == 0 || limit > memblock_end) limit = memblock_end; + if (base + size > limit) { + ret = -EINVAL; + pr_err("Size (%pa) of region at %pa exceeds limit (%pa)\n", + &size, &base, &limit); + goto err; + } + /* Reserve memory */ if (fixed) { if (memblock_is_region_reserved(base, size) || @@ -494,7 +507,7 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, * @pages: Allocated pages. * @count: Number of allocated pages. * - * This function releases memory allocated by alloc_cma(). + * This function releases memory allocated by cma_alloc(). * It returns false when provided pages do not belong to contiguous area and * true otherwise. */ @@ -1895,7 +1895,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, } #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ -#if defined(__HAVE_ARCH_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE) +#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE) static int __gup_device_huge(unsigned long pfn, unsigned long addr, unsigned long end, struct page **pages, int *nr) { diff --git a/mm/maccess.c b/mm/maccess.c index 482d4d670f19..d065736f6b87 100644 --- a/mm/maccess.c +++ b/mm/maccess.c @@ -6,8 +6,20 @@ #include <linux/mm.h> #include <linux/uaccess.h> +static __always_inline long +probe_read_common(void *dst, const void __user *src, size_t size) +{ + long ret; + + pagefault_disable(); + ret = __copy_from_user_inatomic(dst, src, size); + pagefault_enable(); + + return ret ? -EFAULT : 0; +} + /** - * probe_kernel_read(): safely attempt to read from a location + * probe_kernel_read(): safely attempt to read from a kernel-space location * @dst: pointer to the buffer that shall take the data * @src: address to read from * @size: size of the data chunk @@ -30,17 +42,41 @@ long __probe_kernel_read(void *dst, const void *src, size_t size) mm_segment_t old_fs = get_fs(); set_fs(KERNEL_DS); - pagefault_disable(); - ret = __copy_from_user_inatomic(dst, - (__force const void __user *)src, size); - pagefault_enable(); + ret = probe_read_common(dst, (__force const void __user *)src, size); set_fs(old_fs); - return ret ? -EFAULT : 0; + return ret; } EXPORT_SYMBOL_GPL(probe_kernel_read); /** + * probe_user_read(): safely attempt to read from a user-space location + * @dst: pointer to the buffer that shall take the data + * @src: address to read from. This must be a user address. + * @size: size of the data chunk + * + * Safely read from user address @src to the buffer at @dst. If a kernel fault + * happens, handle that and return -EFAULT. + */ + +long __weak probe_user_read(void *dst, const void __user *src, size_t size) + __attribute__((alias("__probe_user_read"))); + +long __probe_user_read(void *dst, const void __user *src, size_t size) +{ + long ret = -EFAULT; + mm_segment_t old_fs = get_fs(); + + set_fs(USER_DS); + if (access_ok(src, size)) + ret = probe_read_common(dst, src, size); + set_fs(old_fs); + + return ret; +} +EXPORT_SYMBOL_GPL(probe_user_read); + +/** * probe_kernel_write(): safely attempt to write to a location * @dst: address to write to * @src: pointer to the data that shall be written @@ -67,6 +103,7 @@ long __probe_kernel_write(void *dst, const void *src, size_t size) } EXPORT_SYMBOL_GPL(probe_kernel_write); + /** * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address. * @dst: Destination address, in kernel space. This buffer must be at @@ -106,3 +143,76 @@ long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) return ret ? -EFAULT : src - unsafe_addr; } + +/** + * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user + * address. + * @dst: Destination address, in kernel space. This buffer must be at + * least @count bytes long. + * @unsafe_addr: Unsafe user address. + * @count: Maximum number of bytes to copy, including the trailing NUL. + * + * Copies a NUL-terminated string from unsafe user address to kernel buffer. + * + * On success, returns the length of the string INCLUDING the trailing NUL. + * + * If access fails, returns -EFAULT (some data may have been copied + * and the trailing NUL added). + * + * If @count is smaller than the length of the string, copies @count-1 bytes, + * sets the last byte of @dst buffer to NUL and returns @count. + */ +long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr, + long count) +{ + mm_segment_t old_fs = get_fs(); + long ret; + + if (unlikely(count <= 0)) + return 0; + + set_fs(USER_DS); + pagefault_disable(); + ret = strncpy_from_user(dst, unsafe_addr, count); + pagefault_enable(); + set_fs(old_fs); + + if (ret >= count) { + ret = count; + dst[ret - 1] = '\0'; + } else if (ret > 0) { + ret++; + } + + return ret; +} + +/** + * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL. + * @unsafe_addr: The string to measure. + * @count: Maximum count (including NUL) + * + * Get the size of a NUL-terminated string in user space without pagefault. + * + * Returns the size of the string INCLUDING the terminating NUL. + * + * If the string is too long, returns a number larger than @count. User + * has to check the return value against "> count". + * On exception (or invalid count), returns 0. + * + * Unlike strnlen_user, this can be used from IRQ handler etc. because + * it disables pagefaults. + */ +long strnlen_unsafe_user(const void __user *unsafe_addr, long count) +{ + mm_segment_t old_fs = get_fs(); + int ret; + + set_fs(USER_DS); + pagefault_disable(); + ret = strnlen_user(unsafe_addr, count); + pagefault_enable(); + set_fs(old_fs); + + return ret; +} diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 249671873aa9..cdbb7a84cb6e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -695,12 +695,15 @@ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) if (mem_cgroup_disabled()) return; - __this_cpu_add(memcg->vmstats_local->stat[idx], val); - x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]); if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) { struct mem_cgroup *mi; + /* + * Batch local counters to keep them in sync with + * the hierarchical ones. + */ + __this_cpu_add(memcg->vmstats_local->stat[idx], x); for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) atomic_long_add(x, &mi->vmstats[idx]); x = 0; @@ -749,13 +752,15 @@ void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, /* Update memcg */ __mod_memcg_state(memcg, idx, val); - /* Update lruvec */ - __this_cpu_add(pn->lruvec_stat_local->count[idx], val); - x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]); if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) { struct mem_cgroup_per_node *pi; + /* + * Batch local counters to keep them in sync with + * the hierarchical ones. + */ + __this_cpu_add(pn->lruvec_stat_local->count[idx], x); for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id)) atomic_long_add(x, &pi->lruvec_stat[idx]); x = 0; @@ -777,12 +782,15 @@ void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, if (mem_cgroup_disabled()) return; - __this_cpu_add(memcg->vmstats_local->events[idx], count); - x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]); if (unlikely(x > MEMCG_CHARGE_BATCH)) { struct mem_cgroup *mi; + /* + * Batch local counters to keep them in sync with + * the hierarchical ones. + */ + __this_cpu_add(memcg->vmstats_local->events[idx], x); for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) atomic_long_add(x, &mi->vmevents[idx]); x = 0; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 6166ba5a15f3..4ebe696138e8 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1734,9 +1734,10 @@ static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1; pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n", &beginpa, &endpa); - } - return ret; + return -EBUSY; + } + return 0; } static int check_cpu_on_node(pg_data_t *pgdat) @@ -1819,19 +1820,9 @@ static void __release_memory_resource(resource_size_t start, } } -/** - * remove_memory - * @nid: the node ID - * @start: physical address of the region to remove - * @size: size of the region to remove - * - * NOTE: The caller must call lock_device_hotplug() to serialize hotplug - * and online/offline operations before this call, as required by - * try_offline_node(). - */ -void __ref __remove_memory(int nid, u64 start, u64 size) +static int __ref try_remove_memory(int nid, u64 start, u64 size) { - int ret; + int rc = 0; BUG_ON(check_hotplug_memory_range(start, size)); @@ -1839,13 +1830,13 @@ void __ref __remove_memory(int nid, u64 start, u64 size) /* * All memory blocks must be offlined before removing memory. Check - * whether all memory blocks in question are offline and trigger a BUG() + * whether all memory blocks in question are offline and return error * if this is not the case. */ - ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, - check_memblock_offlined_cb); - if (ret) - BUG(); + rc = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, + check_memblock_offlined_cb); + if (rc) + goto done; /* remove memmap entry */ firmware_map_remove(start, start + size, "System RAM"); @@ -1857,14 +1848,45 @@ void __ref __remove_memory(int nid, u64 start, u64 size) try_offline_node(nid); +done: mem_hotplug_done(); + return rc; } -void remove_memory(int nid, u64 start, u64 size) +/** + * remove_memory + * @nid: the node ID + * @start: physical address of the region to remove + * @size: size of the region to remove + * + * NOTE: The caller must call lock_device_hotplug() to serialize hotplug + * and online/offline operations before this call, as required by + * try_offline_node(). + */ +void __remove_memory(int nid, u64 start, u64 size) +{ + + /* + * trigger BUG() is some memory is not offlined prior to calling this + * function + */ + if (try_remove_memory(nid, start, size)) + BUG(); +} + +/* + * Remove memory if every memory block is offline, otherwise return -EBUSY is + * some memory is not offline + */ +int remove_memory(int nid, u64 start, u64 size) { + int rc; + lock_device_hotplug(); - __remove_memory(nid, start, size); + rc = try_remove_memory(nid, start, size); unlock_device_hotplug(); + + return rc; } EXPORT_SYMBOL_GPL(remove_memory); #endif /* CONFIG_MEMORY_HOTREMOVE */ diff --git a/mm/nommu.c b/mm/nommu.c index eb3e2e558da1..fed1b6e9c89b 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1261,7 +1261,9 @@ unsigned long do_mmap(struct file *file, add_nommu_region(region); /* clear anonymous mappings that don't ask for uninitialized data */ - if (!vma->vm_file && !(flags & MAP_UNINITIALIZED)) + if (!vma->vm_file && + (!IS_ENABLED(CONFIG_MMAP_ALLOW_UNINITIALIZED) || + !(flags & MAP_UNINITIALIZED))) memset((void *)region->vm_start, 0, region->vm_end - region->vm_start); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 8fd7f45a04eb..e515bfcf7f28 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4102,7 +4102,6 @@ static int __perform_reclaim(gfp_t gfp_mask, unsigned int order, const struct alloc_context *ac) { - struct reclaim_state reclaim_state; int progress; unsigned int noreclaim_flag; unsigned long pflags; @@ -4114,13 +4113,10 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, psi_memstall_enter(&pflags); fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); - reclaim_state.reclaimed_slab = 0; - current->reclaim_state = &reclaim_state; progress = try_to_free_pages(ac->zonelist, order, gfp_mask, ac->nodemask); - current->reclaim_state = NULL; memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(gfp_mask); psi_memstall_leave(&pflags); diff --git a/mm/shmem.c b/mm/shmem.c index f4dce9c8670d..99497cb32e71 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -400,7 +400,7 @@ static bool shmem_confirm_swap(struct address_space *mapping, static int shmem_huge __read_mostly; -#if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) +#if defined(CONFIG_SYSFS) static int shmem_parse_huge(const char *str) { if (!strcmp(str, "never")) @@ -417,7 +417,9 @@ static int shmem_parse_huge(const char *str) return SHMEM_HUGE_FORCE; return -EINVAL; } +#endif +#if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) static const char *shmem_format_huge(int huge) { switch (huge) { diff --git a/mm/slab_common.c b/mm/slab_common.c index 6c49dbb3769e..807490fe217a 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1028,7 +1028,8 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, } struct kmem_cache * -kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init; +kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init = +{ /* initialization for https://bugs.llvm.org/show_bug.cgi?id=42570 */ }; EXPORT_SYMBOL(kmalloc_caches); /* diff --git a/mm/util.c b/mm/util.c index 68575a315dc5..e6351a80f248 100644 --- a/mm/util.c +++ b/mm/util.c @@ -7,6 +7,7 @@ #include <linux/err.h> #include <linux/sched.h> #include <linux/sched/mm.h> +#include <linux/sched/signal.h> #include <linux/sched/task_stack.h> #include <linux/security.h> #include <linux/swap.h> @@ -300,6 +301,80 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) } #endif +/** + * __account_locked_vm - account locked pages to an mm's locked_vm + * @mm: mm to account against + * @pages: number of pages to account + * @inc: %true if @pages should be considered positive, %false if not + * @task: task used to check RLIMIT_MEMLOCK + * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped + * + * Assumes @task and @mm are valid (i.e. at least one reference on each), and + * that mmap_sem is held as writer. + * + * Return: + * * 0 on success + * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded. + */ +int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, + struct task_struct *task, bool bypass_rlim) +{ + unsigned long locked_vm, limit; + int ret = 0; + + lockdep_assert_held_write(&mm->mmap_sem); + + locked_vm = mm->locked_vm; + if (inc) { + if (!bypass_rlim) { + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; + if (locked_vm + pages > limit) + ret = -ENOMEM; + } + if (!ret) + mm->locked_vm = locked_vm + pages; + } else { + WARN_ON_ONCE(pages > locked_vm); + mm->locked_vm = locked_vm - pages; + } + + pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, + (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, + locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + ret ? " - exceeded" : ""); + + return ret; +} +EXPORT_SYMBOL_GPL(__account_locked_vm); + +/** + * account_locked_vm - account locked pages to an mm's locked_vm + * @mm: mm to account against, may be NULL + * @pages: number of pages to account + * @inc: %true if @pages should be considered positive, %false if not + * + * Assumes a non-NULL @mm is valid (i.e. at least one reference on it). + * + * Return: + * * 0 on success, or if mm is NULL + * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded. + */ +int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc) +{ + int ret; + + if (pages == 0 || !mm) + return 0; + + down_write(&mm->mmap_sem); + ret = __account_locked_vm(mm, pages, inc, current, + capable(CAP_IPC_LOCK)); + up_write(&mm->mmap_sem); + + return ret; +} +EXPORT_SYMBOL_GPL(account_locked_vm); + unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff) diff --git a/mm/vmscan.c b/mm/vmscan.c index f8e3dcd527b8..44df66a98f2a 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -131,6 +131,9 @@ struct scan_control { unsigned int file_taken; unsigned int taken; } nr; + + /* for recording the reclaimed slab by now */ + struct reclaim_state reclaim_state; }; #ifdef ARCH_HAS_PREFETCH @@ -238,6 +241,18 @@ static void unregister_memcg_shrinker(struct shrinker *shrinker) } #endif /* CONFIG_MEMCG_KMEM */ +static void set_task_reclaim_state(struct task_struct *task, + struct reclaim_state *rs) +{ + /* Check for an overwrite */ + WARN_ON_ONCE(rs && task->reclaim_state); + + /* Check for the nulling of an already-nulled member */ + WARN_ON_ONCE(!rs && !task->reclaim_state); + + task->reclaim_state = rs; +} + #ifdef CONFIG_MEMCG static bool global_reclaim(struct scan_control *sc) { @@ -3191,11 +3206,13 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) return 1; + set_task_reclaim_state(current, &sc.reclaim_state); trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); nr_reclaimed = do_try_to_free_pages(zonelist, &sc); trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); + set_task_reclaim_state(current, NULL); return nr_reclaimed; } @@ -3218,6 +3235,7 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, }; unsigned long lru_pages; + set_task_reclaim_state(current, &sc.reclaim_state); sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); @@ -3235,7 +3253,9 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); + set_task_reclaim_state(current, NULL); *nr_scanned = sc.nr_scanned; + return sc.nr_reclaimed; } @@ -3262,6 +3282,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, .may_shrinkslab = 1, }; + set_task_reclaim_state(current, &sc.reclaim_state); /* * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't * take care of from where we get pages. So the node where we start the @@ -3282,6 +3303,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, psi_memstall_leave(&pflags); trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); + set_task_reclaim_state(current, NULL); return nr_reclaimed; } @@ -3483,6 +3505,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx) .may_unmap = 1, }; + set_task_reclaim_state(current, &sc.reclaim_state); psi_memstall_enter(&pflags); __fs_reclaim_acquire(); @@ -3664,6 +3687,8 @@ out: snapshot_refaults(NULL, pgdat); __fs_reclaim_release(); psi_memstall_leave(&pflags); + set_task_reclaim_state(current, NULL); + /* * Return the order kswapd stopped reclaiming at as * prepare_kswapd_sleep() takes it into account. If another caller @@ -3787,15 +3812,10 @@ static int kswapd(void *p) unsigned int classzone_idx = MAX_NR_ZONES - 1; pg_data_t *pgdat = (pg_data_t*)p; struct task_struct *tsk = current; - - struct reclaim_state reclaim_state = { - .reclaimed_slab = 0, - }; const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); if (!cpumask_empty(cpumask)) set_cpus_allowed_ptr(tsk, cpumask); - current->reclaim_state = &reclaim_state; /* * Tell the memory management that we're a "memory allocator", @@ -3857,7 +3877,6 @@ kswapd_try_sleep: } tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD); - current->reclaim_state = NULL; return 0; } @@ -3922,7 +3941,6 @@ void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, */ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) { - struct reclaim_state reclaim_state; struct scan_control sc = { .nr_to_reclaim = nr_to_reclaim, .gfp_mask = GFP_HIGHUSER_MOVABLE, @@ -3934,18 +3952,16 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) .hibernation_mode = 1, }; struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); - struct task_struct *p = current; unsigned long nr_reclaimed; unsigned int noreclaim_flag; fs_reclaim_acquire(sc.gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); - reclaim_state.reclaimed_slab = 0; - p->reclaim_state = &reclaim_state; + set_task_reclaim_state(current, &sc.reclaim_state); nr_reclaimed = do_try_to_free_pages(zonelist, &sc); - p->reclaim_state = NULL; + set_task_reclaim_state(current, NULL); memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(sc.gfp_mask); @@ -4110,7 +4126,6 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in /* Minimum pages needed in order to stay on node */ const unsigned long nr_pages = 1 << order; struct task_struct *p = current; - struct reclaim_state reclaim_state; unsigned int noreclaim_flag; struct scan_control sc = { .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), @@ -4135,8 +4150,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in */ noreclaim_flag = memalloc_noreclaim_save(); p->flags |= PF_SWAPWRITE; - reclaim_state.reclaimed_slab = 0; - p->reclaim_state = &reclaim_state; + set_task_reclaim_state(p, &sc.reclaim_state); if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) { /* @@ -4148,7 +4162,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); } - p->reclaim_state = NULL; + set_task_reclaim_state(p, NULL); current->flags &= ~PF_SWAPWRITE; memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(sc.gfp_mask); diff --git a/mm/z3fold.c b/mm/z3fold.c index dfcd69d08c1e..6c72b18d8b9c 100644 --- a/mm/z3fold.c +++ b/mm/z3fold.c @@ -101,6 +101,7 @@ struct z3fold_buddy_slots { * @refcount: reference count for the z3fold page * @work: work_struct for page layout optimization * @slots: pointer to the structure holding buddy slots + * @pool: pointer to the containing pool * @cpu: CPU which this page "belongs" to * @first_chunks: the size of the first buddy in chunks, 0 if free * @middle_chunks: the size of the middle buddy in chunks, 0 if free @@ -114,6 +115,7 @@ struct z3fold_header { struct kref refcount; struct work_struct work; struct z3fold_buddy_slots *slots; + struct z3fold_pool *pool; short cpu; unsigned short first_chunks; unsigned short middle_chunks; @@ -193,8 +195,10 @@ static void compact_page_work(struct work_struct *w); static inline struct z3fold_buddy_slots *alloc_slots(struct z3fold_pool *pool, gfp_t gfp) { - struct z3fold_buddy_slots *slots = kmem_cache_alloc(pool->c_handle, - gfp); + struct z3fold_buddy_slots *slots; + + slots = kmem_cache_alloc(pool->c_handle, + (gfp & ~(__GFP_HIGHMEM | __GFP_MOVABLE))); if (slots) { memset(slots->slot, 0, sizeof(slots->slot)); @@ -320,6 +324,7 @@ static struct z3fold_header *init_z3fold_page(struct page *page, zhdr->start_middle = 0; zhdr->cpu = -1; zhdr->slots = slots; + zhdr->pool = pool; INIT_LIST_HEAD(&zhdr->buddy); INIT_WORK(&zhdr->work, compact_page_work); return zhdr; @@ -426,7 +431,7 @@ static enum buddy handle_to_buddy(unsigned long handle) static inline struct z3fold_pool *zhdr_to_pool(struct z3fold_header *zhdr) { - return slots_to_pool(zhdr->slots); + return zhdr->pool; } static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) @@ -850,7 +855,7 @@ static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp, enum buddy bud; bool can_sleep = gfpflags_allow_blocking(gfp); - if (!size || (gfp & __GFP_HIGHMEM)) + if (!size) return -EINVAL; if (size > PAGE_SIZE) @@ -1345,24 +1350,29 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa zhdr = page_address(page); pool = zhdr_to_pool(zhdr); - if (!trylock_page(page)) - return -EAGAIN; - if (!z3fold_page_trylock(zhdr)) { - unlock_page(page); return -EAGAIN; } if (zhdr->mapped_count != 0) { z3fold_page_unlock(zhdr); - unlock_page(page); return -EBUSY; } + if (work_pending(&zhdr->work)) { + z3fold_page_unlock(zhdr); + return -EAGAIN; + } new_zhdr = page_address(newpage); memcpy(new_zhdr, zhdr, PAGE_SIZE); newpage->private = page->private; page->private = 0; z3fold_page_unlock(zhdr); spin_lock_init(&new_zhdr->page_lock); + INIT_WORK(&new_zhdr->work, compact_page_work); + /* + * z3fold_page_isolate() ensures that new_zhdr->buddy is empty, + * so we only have to reinitialize it. + */ + INIT_LIST_HEAD(&new_zhdr->buddy); new_mapping = page_mapping(page); __ClearPageMovable(page); ClearPagePrivate(page); @@ -1386,7 +1396,6 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work); page_mapcount_reset(page); - unlock_page(page); put_page(page); return 0; } diff --git a/net/ceph/Makefile b/net/ceph/Makefile index db09defe27d0..59d0ba2072de 100644 --- a/net/ceph/Makefile +++ b/net/ceph/Makefile @@ -5,7 +5,7 @@ obj-$(CONFIG_CEPH_LIB) += libceph.o libceph-y := ceph_common.o messenger.o msgpool.o buffer.o pagelist.o \ - mon_client.o \ + mon_client.o decode.o \ cls_lock_client.o \ osd_client.o osdmap.o crush/crush.o crush/mapper.o crush/hash.o \ striper.o \ diff --git a/net/ceph/cls_lock_client.c b/net/ceph/cls_lock_client.c index 4cc28541281b..17447c19d937 100644 --- a/net/ceph/cls_lock_client.c +++ b/net/ceph/cls_lock_client.c @@ -6,6 +6,7 @@ #include <linux/ceph/cls_lock_client.h> #include <linux/ceph/decode.h> +#include <linux/ceph/libceph.h> /** * ceph_cls_lock - grab rados lock for object @@ -264,8 +265,11 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker) return ret; *p += sizeof(struct ceph_timespec); /* skip expiration */ - ceph_decode_copy(p, &locker->info.addr, sizeof(locker->info.addr)); - ceph_decode_addr(&locker->info.addr); + + ret = ceph_decode_entity_addr(p, end, &locker->info.addr); + if (ret) + return ret; + len = ceph_decode_32(p); *p += len; /* skip description */ @@ -360,7 +364,7 @@ int ceph_cls_lock_info(struct ceph_osd_client *osdc, dout("%s lock_name %s\n", __func__, lock_name); ret = ceph_osdc_call(osdc, oid, oloc, "lock", "get_info", CEPH_OSD_FLAG_READ, get_info_op_page, - get_info_op_buf_size, reply_page, &reply_len); + get_info_op_buf_size, &reply_page, &reply_len); dout("%s: status %d\n", __func__, ret); if (ret >= 0) { @@ -375,3 +379,47 @@ int ceph_cls_lock_info(struct ceph_osd_client *osdc, return ret; } EXPORT_SYMBOL(ceph_cls_lock_info); + +int ceph_cls_assert_locked(struct ceph_osd_request *req, int which, + char *lock_name, u8 type, char *cookie, char *tag) +{ + int assert_op_buf_size; + int name_len = strlen(lock_name); + int cookie_len = strlen(cookie); + int tag_len = strlen(tag); + struct page **pages; + void *p, *end; + int ret; + + assert_op_buf_size = name_len + sizeof(__le32) + + cookie_len + sizeof(__le32) + + tag_len + sizeof(__le32) + + sizeof(u8) + CEPH_ENCODING_START_BLK_LEN; + if (assert_op_buf_size > PAGE_SIZE) + return -E2BIG; + + ret = osd_req_op_cls_init(req, which, "lock", "assert_locked"); + if (ret) + return ret; + + pages = ceph_alloc_page_vector(1, GFP_NOIO); + if (IS_ERR(pages)) + return PTR_ERR(pages); + + p = page_address(pages[0]); + end = p + assert_op_buf_size; + + /* encode cls_lock_assert_op struct */ + ceph_start_encoding(&p, 1, 1, + assert_op_buf_size - CEPH_ENCODING_START_BLK_LEN); + ceph_encode_string(&p, end, lock_name, name_len); + ceph_encode_8(&p, type); + ceph_encode_string(&p, end, cookie, cookie_len); + ceph_encode_string(&p, end, tag, tag_len); + WARN_ON(p != end); + + osd_req_op_cls_request_data_pages(req, which, pages, assert_op_buf_size, + 0, false, true); + return 0; +} +EXPORT_SYMBOL(ceph_cls_assert_locked); diff --git a/net/ceph/decode.c b/net/ceph/decode.c new file mode 100644 index 000000000000..eea529595a7a --- /dev/null +++ b/net/ceph/decode.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/ceph/decode.h> + +static int +ceph_decode_entity_addr_versioned(void **p, void *end, + struct ceph_entity_addr *addr) +{ + int ret; + u8 struct_v; + u32 struct_len, addr_len; + void *struct_end; + + ret = ceph_start_decoding(p, end, 1, "entity_addr_t", &struct_v, + &struct_len); + if (ret) + goto bad; + + ret = -EINVAL; + struct_end = *p + struct_len; + + ceph_decode_copy_safe(p, end, &addr->type, sizeof(addr->type), bad); + + ceph_decode_copy_safe(p, end, &addr->nonce, sizeof(addr->nonce), bad); + + ceph_decode_32_safe(p, end, addr_len, bad); + if (addr_len > sizeof(addr->in_addr)) + goto bad; + + memset(&addr->in_addr, 0, sizeof(addr->in_addr)); + if (addr_len) { + ceph_decode_copy_safe(p, end, &addr->in_addr, addr_len, bad); + + addr->in_addr.ss_family = + le16_to_cpu((__force __le16)addr->in_addr.ss_family); + } + + /* Advance past anything the client doesn't yet understand */ + *p = struct_end; + ret = 0; +bad: + return ret; +} + +static int +ceph_decode_entity_addr_legacy(void **p, void *end, + struct ceph_entity_addr *addr) +{ + int ret = -EINVAL; + + /* Skip rest of type field */ + ceph_decode_skip_n(p, end, 3, bad); + + /* + * Clients that don't support ADDR2 always send TYPE_NONE, change it + * to TYPE_LEGACY for forward compatibility. + */ + addr->type = CEPH_ENTITY_ADDR_TYPE_LEGACY; + ceph_decode_copy_safe(p, end, &addr->nonce, sizeof(addr->nonce), bad); + memset(&addr->in_addr, 0, sizeof(addr->in_addr)); + ceph_decode_copy_safe(p, end, &addr->in_addr, + sizeof(addr->in_addr), bad); + addr->in_addr.ss_family = + be16_to_cpu((__force __be16)addr->in_addr.ss_family); + ret = 0; +bad: + return ret; +} + +int +ceph_decode_entity_addr(void **p, void *end, struct ceph_entity_addr *addr) +{ + u8 marker; + + ceph_decode_8_safe(p, end, marker, bad); + if (marker == 1) + return ceph_decode_entity_addr_versioned(p, end, addr); + else if (marker == 0) + return ceph_decode_entity_addr_legacy(p, end, addr); +bad: + return -EINVAL; +} +EXPORT_SYMBOL(ceph_decode_entity_addr); + diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index a33402c99321..962f521c863e 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -199,12 +199,14 @@ const char *ceph_pr_addr(const struct ceph_entity_addr *addr) switch (ss.ss_family) { case AF_INET: - snprintf(s, MAX_ADDR_STR_LEN, "%pI4:%hu", &in4->sin_addr, + snprintf(s, MAX_ADDR_STR_LEN, "(%d)%pI4:%hu", + le32_to_cpu(addr->type), &in4->sin_addr, ntohs(in4->sin_port)); break; case AF_INET6: - snprintf(s, MAX_ADDR_STR_LEN, "[%pI6c]:%hu", &in6->sin6_addr, + snprintf(s, MAX_ADDR_STR_LEN, "(%d)[%pI6c]:%hu", + le32_to_cpu(addr->type), &in6->sin6_addr, ntohs(in6->sin6_port)); break; @@ -220,7 +222,7 @@ EXPORT_SYMBOL(ceph_pr_addr); static void encode_my_addr(struct ceph_messenger *msgr) { memcpy(&msgr->my_enc_addr, &msgr->inst.addr, sizeof(msgr->my_enc_addr)); - ceph_encode_addr(&msgr->my_enc_addr); + ceph_encode_banner_addr(&msgr->my_enc_addr); } /* @@ -1732,12 +1734,14 @@ static int read_partial_banner(struct ceph_connection *con) ret = read_partial(con, end, size, &con->actual_peer_addr); if (ret <= 0) goto out; + ceph_decode_banner_addr(&con->actual_peer_addr); size = sizeof (con->peer_addr_for_me); end += size; ret = read_partial(con, end, size, &con->peer_addr_for_me); if (ret <= 0) goto out; + ceph_decode_banner_addr(&con->peer_addr_for_me); out: return ret; @@ -1981,6 +1985,7 @@ int ceph_parse_ips(const char *c, const char *end, } addr_set_port(&addr[i], port); + addr[i].type = CEPH_ENTITY_ADDR_TYPE_LEGACY; dout("parse_ips got %s\n", ceph_pr_addr(&addr[i])); @@ -2011,9 +2016,6 @@ static int process_banner(struct ceph_connection *con) if (verify_hello(con) < 0) return -1; - ceph_decode_addr(&con->actual_peer_addr); - ceph_decode_addr(&con->peer_addr_for_me); - /* * Make sure the other end is who we wanted. note that the other * end may not yet know their ip address, so if it's 0.0.0.0, give diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 895679d3529b..0520bf9825aa 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -39,7 +39,7 @@ static int __validate_auth(struct ceph_mon_client *monc); /* * Decode a monmap blob (e.g., during mount). */ -struct ceph_monmap *ceph_monmap_decode(void *p, void *end) +static struct ceph_monmap *ceph_monmap_decode(void *p, void *end) { struct ceph_monmap *m = NULL; int i, err = -EINVAL; @@ -50,7 +50,7 @@ struct ceph_monmap *ceph_monmap_decode(void *p, void *end) ceph_decode_32_safe(&p, end, len, bad); ceph_decode_need(&p, end, len, bad); - dout("monmap_decode %p %p len %d\n", p, end, (int)(end-p)); + dout("monmap_decode %p %p len %d (%d)\n", p, end, len, (int)(end-p)); p += sizeof(u16); /* skip version */ ceph_decode_need(&p, end, sizeof(fsid) + 2*sizeof(u32), bad); @@ -58,7 +58,6 @@ struct ceph_monmap *ceph_monmap_decode(void *p, void *end) epoch = ceph_decode_32(&p); num_mon = ceph_decode_32(&p); - ceph_decode_need(&p, end, num_mon*sizeof(m->mon_inst[0]), bad); if (num_mon > CEPH_MAX_MON) goto bad; @@ -68,17 +67,22 @@ struct ceph_monmap *ceph_monmap_decode(void *p, void *end) m->fsid = fsid; m->epoch = epoch; m->num_mon = num_mon; - ceph_decode_copy(&p, m->mon_inst, num_mon*sizeof(m->mon_inst[0])); - for (i = 0; i < num_mon; i++) - ceph_decode_addr(&m->mon_inst[i].addr); - + for (i = 0; i < num_mon; ++i) { + struct ceph_entity_inst *inst = &m->mon_inst[i]; + + /* copy name portion */ + ceph_decode_copy_safe(&p, end, &inst->name, + sizeof(inst->name), bad); + err = ceph_decode_entity_addr(&p, end, &inst->addr); + if (err) + goto bad; + } dout("monmap_decode epoch %d, num_mon %d\n", m->epoch, m->num_mon); for (i = 0; i < m->num_mon; i++) dout("monmap_decode mon%d is %s\n", i, ceph_pr_addr(&m->mon_inst[i].addr)); return m; - bad: dout("monmap_decode failed with %d\n", err); kfree(m); @@ -469,6 +473,7 @@ static void ceph_monc_handle_map(struct ceph_mon_client *monc, if (IS_ERR(monmap)) { pr_err("problem decoding monmap, %d\n", (int)PTR_ERR(monmap)); + ceph_msg_dump(msg); goto out; } diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 9a8eca5eda65..0b2df09b2554 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -171,14 +171,6 @@ static void ceph_osd_data_bvecs_init(struct ceph_osd_data *osd_data, osd_data->num_bvecs = num_bvecs; } -#define osd_req_op_data(oreq, whch, typ, fld) \ -({ \ - struct ceph_osd_request *__oreq = (oreq); \ - unsigned int __whch = (whch); \ - BUG_ON(__whch >= __oreq->r_num_ops); \ - &__oreq->r_ops[__whch].typ.fld; \ -}) - static struct ceph_osd_data * osd_req_op_raw_data_in(struct ceph_osd_request *osd_req, unsigned int which) { @@ -478,7 +470,7 @@ static void request_release_checks(struct ceph_osd_request *req) { WARN_ON(!RB_EMPTY_NODE(&req->r_node)); WARN_ON(!RB_EMPTY_NODE(&req->r_mc_node)); - WARN_ON(!list_empty(&req->r_unsafe_item)); + WARN_ON(!list_empty(&req->r_private_item)); WARN_ON(req->r_osd); } @@ -538,7 +530,7 @@ static void request_init(struct ceph_osd_request *req) init_completion(&req->r_completion); RB_CLEAR_NODE(&req->r_node); RB_CLEAR_NODE(&req->r_mc_node); - INIT_LIST_HEAD(&req->r_unsafe_item); + INIT_LIST_HEAD(&req->r_private_item); target_init(&req->r_t); } @@ -4914,20 +4906,26 @@ static int decode_watcher(void **p, void *end, struct ceph_watch_item *item) ret = ceph_start_decoding(p, end, 2, "watch_item_t", &struct_v, &struct_len); if (ret) - return ret; + goto bad; + + ret = -EINVAL; + ceph_decode_copy_safe(p, end, &item->name, sizeof(item->name), bad); + ceph_decode_64_safe(p, end, item->cookie, bad); + ceph_decode_skip_32(p, end, bad); /* skip timeout seconds */ - ceph_decode_copy(p, &item->name, sizeof(item->name)); - item->cookie = ceph_decode_64(p); - *p += 4; /* skip timeout_seconds */ if (struct_v >= 2) { - ceph_decode_copy(p, &item->addr, sizeof(item->addr)); - ceph_decode_addr(&item->addr); + ret = ceph_decode_entity_addr(p, end, &item->addr); + if (ret) + goto bad; + } else { + ret = 0; } dout("%s %s%llu cookie %llu addr %s\n", __func__, ENTITY_NAME(item->name), item->cookie, ceph_pr_addr(&item->addr)); - return 0; +bad: + return ret; } static int decode_watchers(void **p, void *end, @@ -5044,12 +5042,12 @@ int ceph_osdc_call(struct ceph_osd_client *osdc, const char *class, const char *method, unsigned int flags, struct page *req_page, size_t req_len, - struct page *resp_page, size_t *resp_len) + struct page **resp_pages, size_t *resp_len) { struct ceph_osd_request *req; int ret; - if (req_len > PAGE_SIZE || (resp_page && *resp_len > PAGE_SIZE)) + if (req_len > PAGE_SIZE) return -E2BIG; req = ceph_osdc_alloc_request(osdc, NULL, 1, false, GFP_NOIO); @@ -5067,8 +5065,8 @@ int ceph_osdc_call(struct ceph_osd_client *osdc, if (req_page) osd_req_op_cls_request_data_pages(req, 0, &req_page, req_len, 0, false, false); - if (resp_page) - osd_req_op_cls_response_data_pages(req, 0, &resp_page, + if (resp_pages) + osd_req_op_cls_response_data_pages(req, 0, resp_pages, *resp_len, 0, false, false); ret = ceph_osdc_alloc_messages(req, GFP_NOIO); @@ -5079,7 +5077,7 @@ int ceph_osdc_call(struct ceph_osd_client *osdc, ret = ceph_osdc_wait_request(osdc, req); if (ret >= 0) { ret = req->r_ops[0].rval; - if (resp_page) + if (resp_pages) *resp_len = req->r_ops[0].outdata_len; } diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index 48a31dc9161c..90437906b7bc 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -1489,11 +1489,9 @@ static int osdmap_decode(void **p, void *end, struct ceph_osdmap *map) /* osd_state, osd_weight, osd_addrs->client_addr */ ceph_decode_need(p, end, 3*sizeof(u32) + - map->max_osd*((struct_v >= 5 ? sizeof(u32) : - sizeof(u8)) + - sizeof(*map->osd_weight) + - sizeof(*map->osd_addr)), e_inval); - + map->max_osd*(struct_v >= 5 ? sizeof(u32) : + sizeof(u8)) + + sizeof(*map->osd_weight), e_inval); if (ceph_decode_32(p) != map->max_osd) goto e_inval; @@ -1514,9 +1512,11 @@ static int osdmap_decode(void **p, void *end, struct ceph_osdmap *map) if (ceph_decode_32(p) != map->max_osd) goto e_inval; - ceph_decode_copy(p, map->osd_addr, map->max_osd*sizeof(*map->osd_addr)); - for (i = 0; i < map->max_osd; i++) - ceph_decode_addr(&map->osd_addr[i]); + for (i = 0; i < map->max_osd; i++) { + err = ceph_decode_entity_addr(p, end, &map->osd_addr[i]); + if (err) + goto bad; + } /* pg_temp */ err = decode_pg_temp(p, end, map); @@ -1618,12 +1618,17 @@ static int decode_new_up_state_weight(void **p, void *end, u8 struct_v, void *new_state; void *new_weight_end; u32 len; + int i; new_up_client = *p; ceph_decode_32_safe(p, end, len, e_inval); - len *= sizeof(u32) + sizeof(struct ceph_entity_addr); - ceph_decode_need(p, end, len, e_inval); - *p += len; + for (i = 0; i < len; ++i) { + struct ceph_entity_addr addr; + + ceph_decode_skip_32(p, end, e_inval); + if (ceph_decode_entity_addr(p, end, &addr)) + goto e_inval; + } new_state = *p; ceph_decode_32_safe(p, end, len, e_inval); @@ -1699,9 +1704,9 @@ static int decode_new_up_state_weight(void **p, void *end, u8 struct_v, struct ceph_entity_addr addr; osd = ceph_decode_32(p); - ceph_decode_copy(p, &addr, sizeof(addr)); - ceph_decode_addr(&addr); BUG_ON(osd >= map->max_osd); + if (ceph_decode_entity_addr(p, end, &addr)) + goto e_inval; pr_info("osd%d up\n", osd); map->osd_state[osd] |= CEPH_OSD_EXISTS | CEPH_OSD_UP; map->osd_addr[osd] = addr; diff --git a/net/ceph/pagevec.c b/net/ceph/pagevec.c index 74cafc0142ea..64305e7056a1 100644 --- a/net/ceph/pagevec.c +++ b/net/ceph/pagevec.c @@ -10,39 +10,6 @@ #include <linux/ceph/libceph.h> -/* - * build a vector of user pages - */ -struct page **ceph_get_direct_page_vector(const void __user *data, - int num_pages, bool write_page) -{ - struct page **pages; - int got = 0; - int rc = 0; - - pages = kmalloc_array(num_pages, sizeof(*pages), GFP_NOFS); - if (!pages) - return ERR_PTR(-ENOMEM); - - while (got < num_pages) { - rc = get_user_pages_fast( - (unsigned long)data + ((unsigned long)got * PAGE_SIZE), - num_pages - got, write_page ? FOLL_WRITE : 0, pages + got); - if (rc < 0) - break; - BUG_ON(rc == 0); - got += rc; - } - if (rc < 0) - goto fail; - return pages; - -fail: - ceph_put_page_vector(pages, got, false); - return ERR_PTR(rc); -} -EXPORT_SYMBOL(ceph_get_direct_page_vector); - void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty) { int i; diff --git a/net/ceph/striper.c b/net/ceph/striper.c index c36462dc86b7..3b3fa75d1189 100644 --- a/net/ceph/striper.c +++ b/net/ceph/striper.c @@ -259,3 +259,20 @@ int ceph_extent_to_file(struct ceph_file_layout *l, return 0; } EXPORT_SYMBOL(ceph_extent_to_file); + +u64 ceph_get_num_objects(struct ceph_file_layout *l, u64 size) +{ + u64 period = (u64)l->stripe_count * l->object_size; + u64 num_periods = DIV64_U64_ROUND_UP(size, period); + u64 remainder_bytes; + u64 remainder_objs = 0; + + div64_u64_rem(size, period, &remainder_bytes); + if (remainder_bytes > 0 && + remainder_bytes < (u64)l->stripe_count * l->stripe_unit) + remainder_objs = l->stripe_count - + DIV_ROUND_UP_ULL(remainder_bytes, l->stripe_unit); + + return num_periods * l->stripe_count - remainder_objs; +} +EXPORT_SYMBOL(ceph_get_num_objects); diff --git a/samples/vfio-mdev/mbochs.c b/samples/vfio-mdev/mbochs.c index b038aa9f5a70..ac5c8c17b1ff 100644 --- a/samples/vfio-mdev/mbochs.c +++ b/samples/vfio-mdev/mbochs.c @@ -1185,9 +1185,6 @@ static long mbochs_ioctl(struct mdev_device *mdev, unsigned int cmd, { int ret = 0; unsigned long minsz, outsz; - struct mdev_state *mdev_state; - - mdev_state = mdev_get_drvdata(mdev); switch (cmd) { case VFIO_DEVICE_GET_INFO: diff --git a/samples/vfio-mdev/mtty.c b/samples/vfio-mdev/mtty.c index ba7ef53c5f6a..92e770a06ea2 100644 --- a/samples/vfio-mdev/mtty.c +++ b/samples/vfio-mdev/mtty.c @@ -68,7 +68,7 @@ * Global Structures */ -struct mtty_dev { +static struct mtty_dev { dev_t vd_devt; struct class *vd_class; struct cdev vd_cdev; @@ -84,7 +84,7 @@ struct mdev_region_info { }; #if defined(DEBUG_REGS) -const char *wr_reg[] = { +static const char *wr_reg[] = { "TX", "IER", "FCR", @@ -95,7 +95,7 @@ const char *wr_reg[] = { "SCR" }; -const char *rd_reg[] = { +static const char *rd_reg[] = { "RX", "IER", "IIR", @@ -143,8 +143,8 @@ struct mdev_state { int nr_ports; }; -struct mutex mdev_list_lock; -struct list_head mdev_devices_list; +static struct mutex mdev_list_lock; +static struct list_head mdev_devices_list; static const struct file_operations vd_fops = { .owner = THIS_MODULE, @@ -167,7 +167,7 @@ static struct mdev_state *find_mdev_state_by_uuid(const guid_t *uuid) return NULL; } -void dump_buffer(u8 *buf, uint32_t count) +static void dump_buffer(u8 *buf, uint32_t count) { #if defined(DEBUG) int i; @@ -723,7 +723,7 @@ accessfailed: return ret; } -int mtty_create(struct kobject *kobj, struct mdev_device *mdev) +static int mtty_create(struct kobject *kobj, struct mdev_device *mdev) { struct mdev_state *mdev_state; char name[MTTY_STRING_LEN]; @@ -773,7 +773,7 @@ int mtty_create(struct kobject *kobj, struct mdev_device *mdev) return 0; } -int mtty_remove(struct mdev_device *mdev) +static int mtty_remove(struct mdev_device *mdev) { struct mdev_state *mds, *tmp_mds; struct mdev_state *mdev_state = mdev_get_drvdata(mdev); @@ -795,7 +795,7 @@ int mtty_remove(struct mdev_device *mdev) return ret; } -int mtty_reset(struct mdev_device *mdev) +static int mtty_reset(struct mdev_device *mdev) { struct mdev_state *mdev_state; @@ -811,8 +811,8 @@ int mtty_reset(struct mdev_device *mdev) return 0; } -ssize_t mtty_read(struct mdev_device *mdev, char __user *buf, size_t count, - loff_t *ppos) +static ssize_t mtty_read(struct mdev_device *mdev, char __user *buf, + size_t count, loff_t *ppos) { unsigned int done = 0; int ret; @@ -870,7 +870,7 @@ read_err: return -EFAULT; } -ssize_t mtty_write(struct mdev_device *mdev, const char __user *buf, +static ssize_t mtty_write(struct mdev_device *mdev, const char __user *buf, size_t count, loff_t *ppos) { unsigned int done = 0; @@ -1063,7 +1063,7 @@ static int mtty_trigger_interrupt(const guid_t *uuid) return ret; } -int mtty_get_region_info(struct mdev_device *mdev, +static int mtty_get_region_info(struct mdev_device *mdev, struct vfio_region_info *region_info, u16 *cap_type_id, void **cap_type) { @@ -1112,7 +1112,8 @@ int mtty_get_region_info(struct mdev_device *mdev, return 0; } -int mtty_get_irq_info(struct mdev_device *mdev, struct vfio_irq_info *irq_info) +static int mtty_get_irq_info(struct mdev_device *mdev, + struct vfio_irq_info *irq_info) { switch (irq_info->index) { case VFIO_PCI_INTX_IRQ_INDEX: @@ -1136,7 +1137,7 @@ int mtty_get_irq_info(struct mdev_device *mdev, struct vfio_irq_info *irq_info) return 0; } -int mtty_get_device_info(struct mdev_device *mdev, +static int mtty_get_device_info(struct mdev_device *mdev, struct vfio_device_info *dev_info) { dev_info->flags = VFIO_DEVICE_FLAGS_PCI; @@ -1268,13 +1269,13 @@ static long mtty_ioctl(struct mdev_device *mdev, unsigned int cmd, return -ENOTTY; } -int mtty_open(struct mdev_device *mdev) +static int mtty_open(struct mdev_device *mdev) { pr_info("%s\n", __func__); return 0; } -void mtty_close(struct mdev_device *mdev) +static void mtty_close(struct mdev_device *mdev) { pr_info("%s\n", __func__); } @@ -1298,7 +1299,7 @@ static const struct attribute_group mtty_dev_group = { .attrs = mtty_dev_attrs, }; -const struct attribute_group *mtty_dev_groups[] = { +static const struct attribute_group *mtty_dev_groups[] = { &mtty_dev_group, NULL, }; @@ -1325,7 +1326,7 @@ static const struct attribute_group mdev_dev_group = { .attrs = mdev_dev_attrs, }; -const struct attribute_group *mdev_dev_groups[] = { +static const struct attribute_group *mdev_dev_groups[] = { &mdev_dev_group, NULL, }; @@ -1347,7 +1348,7 @@ name_show(struct kobject *kobj, struct device *dev, char *buf) return -EINVAL; } -MDEV_TYPE_ATTR_RO(name); +static MDEV_TYPE_ATTR_RO(name); static ssize_t available_instances_show(struct kobject *kobj, struct device *dev, char *buf) @@ -1375,7 +1376,7 @@ available_instances_show(struct kobject *kobj, struct device *dev, char *buf) return sprintf(buf, "%d\n", (MAX_MTTYS - used)/ports); } -MDEV_TYPE_ATTR_RO(available_instances); +static MDEV_TYPE_ATTR_RO(available_instances); static ssize_t device_api_show(struct kobject *kobj, struct device *dev, @@ -1384,7 +1385,7 @@ static ssize_t device_api_show(struct kobject *kobj, struct device *dev, return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); } -MDEV_TYPE_ATTR_RO(device_api); +static MDEV_TYPE_ATTR_RO(device_api); static struct attribute *mdev_types_attrs[] = { &mdev_type_attr_name.attr, @@ -1403,7 +1404,7 @@ static struct attribute_group mdev_type_group2 = { .attrs = mdev_types_attrs, }; -struct attribute_group *mdev_type_groups[] = { +static struct attribute_group *mdev_type_groups[] = { &mdev_type_group1, &mdev_type_group2, NULL, diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index a6d436809bf5..93a7edfe0f05 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -6639,6 +6639,12 @@ sub process { "unknown module license " . $extracted_string . "\n" . $herecurr); } } + +# check for sysctl duplicate constants + if ($line =~ /\.extra[12]\s*=\s*&(zero|one|int_max)\b/) { + WARN("DUPLICATED_SYSCTL_CONST", + "duplicated sysctl range checking value '$1', consider using the shared one in include/linux/sysctl.h\n" . $herecurr); + } } # If we have no input at all, then there is nothing to report on diff --git a/scripts/gdb/linux/device.py b/scripts/gdb/linux/device.py new file mode 100644 index 000000000000..16376c5cfec6 --- /dev/null +++ b/scripts/gdb/linux/device.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) NXP 2019 + +import gdb + +from linux.utils import CachedType +from linux.utils import container_of +from linux.lists import list_for_each_entry + + +device_private_type = CachedType('struct device_private') +device_type = CachedType('struct device') + +subsys_private_type = CachedType('struct subsys_private') +kobject_type = CachedType('struct kobject') +kset_type = CachedType('struct kset') + +bus_type = CachedType('struct bus_type') +class_type = CachedType('struct class') + + +def dev_name(dev): + dev_init_name = dev['init_name'] + if dev_init_name: + return dev_init_name.string() + return dev['kobj']['name'].string() + + +def kset_for_each_object(kset): + return list_for_each_entry(kset['list'], + kobject_type.get_type().pointer(), "entry") + + +def for_each_bus(): + for kobj in kset_for_each_object(gdb.parse_and_eval('bus_kset')): + subsys = container_of(kobj, kset_type.get_type().pointer(), 'kobj') + subsys_priv = container_of(subsys, subsys_private_type.get_type().pointer(), 'subsys') + yield subsys_priv['bus'] + + +def for_each_class(): + for kobj in kset_for_each_object(gdb.parse_and_eval('class_kset')): + subsys = container_of(kobj, kset_type.get_type().pointer(), 'kobj') + subsys_priv = container_of(subsys, subsys_private_type.get_type().pointer(), 'subsys') + yield subsys_priv['class'] + + +def get_bus_by_name(name): + for item in for_each_bus(): + if item['name'].string() == name: + return item + raise gdb.GdbError("Can't find bus type {!r}".format(name)) + + +def get_class_by_name(name): + for item in for_each_class(): + if item['name'].string() == name: + return item + raise gdb.GdbError("Can't find device class {!r}".format(name)) + + +klist_type = CachedType('struct klist') +klist_node_type = CachedType('struct klist_node') + + +def klist_for_each(klist): + return list_for_each_entry(klist['k_list'], + klist_node_type.get_type().pointer(), 'n_node') + + +def bus_for_each_device(bus): + for kn in klist_for_each(bus['p']['klist_devices']): + dp = container_of(kn, device_private_type.get_type().pointer(), 'knode_bus') + yield dp['device'] + + +def class_for_each_device(cls): + for kn in klist_for_each(cls['p']['klist_devices']): + dp = container_of(kn, device_private_type.get_type().pointer(), 'knode_class') + yield dp['device'] + + +def device_for_each_child(dev): + for kn in klist_for_each(dev['p']['klist_children']): + dp = container_of(kn, device_private_type.get_type().pointer(), 'knode_parent') + yield dp['device'] + + +def _show_device(dev, level=0, recursive=False): + gdb.write('{}dev {}:\t{}\n'.format('\t' * level, dev_name(dev), dev)) + if recursive: + for child in device_for_each_child(dev): + _show_device(child, level + 1, recursive) + + +class LxDeviceListBus(gdb.Command): + '''Print devices on a bus (or all buses if not specified)''' + + def __init__(self): + super(LxDeviceListBus, self).__init__('lx-device-list-bus', gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + if not arg: + for bus in for_each_bus(): + gdb.write('bus {}:\t{}\n'.format(bus['name'].string(), bus)) + for dev in bus_for_each_device(bus): + _show_device(dev, level=1) + else: + bus = get_bus_by_name(arg) + if not bus: + raise gdb.GdbError("Can't find bus {!r}".format(arg)) + for dev in bus_for_each_device(bus): + _show_device(dev) + + +class LxDeviceListClass(gdb.Command): + '''Print devices in a class (or all classes if not specified)''' + + def __init__(self): + super(LxDeviceListClass, self).__init__('lx-device-list-class', gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + if not arg: + for cls in for_each_class(): + gdb.write("class {}:\t{}\n".format(cls['name'].string(), cls)) + for dev in class_for_each_device(cls): + _show_device(dev, level=1) + else: + cls = get_class_by_name(arg) + for dev in class_for_each_device(cls): + _show_device(dev) + + +class LxDeviceListTree(gdb.Command): + '''Print a device and its children recursively''' + + def __init__(self): + super(LxDeviceListTree, self).__init__('lx-device-list-tree', gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + if not arg: + raise gdb.GdbError('Please provide pointer to struct device') + dev = gdb.parse_and_eval(arg) + if dev.type != device_type.get_type().pointer(): + raise gdb.GdbError('Please provide pointer to struct device') + _show_device(dev, level=0, recursive=True) + + +class LxDeviceFindByBusName(gdb.Function): + '''Find struct device by bus and name (both strings)''' + + def __init__(self): + super(LxDeviceFindByBusName, self).__init__('lx_device_find_by_bus_name') + + def invoke(self, bus, name): + name = name.string() + bus = get_bus_by_name(bus.string()) + for dev in bus_for_each_device(bus): + if dev_name(dev) == name: + return dev + + +class LxDeviceFindByClassName(gdb.Function): + '''Find struct device by class and name (both strings)''' + + def __init__(self): + super(LxDeviceFindByClassName, self).__init__('lx_device_find_by_class_name') + + def invoke(self, cls, name): + name = name.string() + cls = get_class_by_name(cls.string()) + for dev in class_for_each_device(cls): + if dev_name(dev) == name: + return dev + + +LxDeviceListBus() +LxDeviceListClass() +LxDeviceListTree() +LxDeviceFindByBusName() +LxDeviceFindByClassName() diff --git a/scripts/gdb/linux/genpd.py b/scripts/gdb/linux/genpd.py new file mode 100644 index 000000000000..6ca93bd2949e --- /dev/null +++ b/scripts/gdb/linux/genpd.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) NXP 2019 + +import gdb +import sys + +from linux.utils import CachedType +from linux.lists import list_for_each_entry + +generic_pm_domain_type = CachedType('struct generic_pm_domain') +pm_domain_data_type = CachedType('struct pm_domain_data') +device_link_type = CachedType('struct device_link') + + +def kobject_get_path(kobj): + path = kobj['name'].string() + parent = kobj['parent'] + if parent: + path = kobject_get_path(parent) + '/' + path + return path + + +def rtpm_status_str(dev): + if dev['power']['runtime_error']: + return 'error' + if dev['power']['disable_depth']: + return 'unsupported' + _RPM_STATUS_LOOKUP = [ + "active", + "resuming", + "suspended", + "suspending" + ] + return _RPM_STATUS_LOOKUP[dev['power']['runtime_status']] + + +class LxGenPDSummary(gdb.Command): + '''Print genpd summary + +Output is similar to /sys/kernel/debug/pm_genpd/pm_genpd_summary''' + + def __init__(self): + super(LxGenPDSummary, self).__init__('lx-genpd-summary', gdb.COMMAND_DATA) + + def summary_one(self, genpd): + if genpd['status'] == 0: + status_string = 'on' + else: + status_string = 'off-{}'.format(genpd['state_idx']) + + slave_names = [] + for link in list_for_each_entry( + genpd['master_links'], + device_link_type.get_type().pointer(), + 'master_node'): + slave_names.apend(link['slave']['name']) + + gdb.write('%-30s %-15s %s\n' % ( + genpd['name'].string(), + status_string, + ', '.join(slave_names))) + + # Print devices in domain + for pm_data in list_for_each_entry(genpd['dev_list'], + pm_domain_data_type.get_type().pointer(), + 'list_node'): + dev = pm_data['dev'] + kobj_path = kobject_get_path(dev['kobj']) + gdb.write(' %-50s %s\n' % (kobj_path, rtpm_status_str(dev))) + + def invoke(self, arg, from_tty): + gdb.write('domain status slaves\n'); + gdb.write(' /device runtime status\n'); + gdb.write('----------------------------------------------------------------------\n'); + for genpd in list_for_each_entry( + gdb.parse_and_eval('&gpd_list'), + generic_pm_domain_type.get_type().pointer(), + 'gpd_list_node'): + self.summary_one(genpd) + + +LxGenPDSummary() diff --git a/scripts/gdb/vmlinux-gdb.py b/scripts/gdb/vmlinux-gdb.py index eff5a48ac026..4136dc2c59df 100644 --- a/scripts/gdb/vmlinux-gdb.py +++ b/scripts/gdb/vmlinux-gdb.py @@ -35,3 +35,5 @@ else: import linux.constants import linux.timerlist import linux.clk + import linux.genpd + import linux.device diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl index c1c088ef1420..5ef59214c555 100755 --- a/scripts/get_maintainer.pl +++ b/scripts/get_maintainer.pl @@ -27,6 +27,7 @@ my $email_usename = 1; my $email_maintainer = 1; my $email_reviewer = 1; my $email_list = 1; +my $email_moderated_list = 1; my $email_subscriber_list = 0; my $email_git_penguin_chiefs = 0; my $email_git = 0; @@ -248,6 +249,7 @@ if (!GetOptions( 'r!' => \$email_reviewer, 'n!' => \$email_usename, 'l!' => \$email_list, + 'moderated!' => \$email_moderated_list, 's!' => \$email_subscriber_list, 'multiline!' => \$output_multiline, 'roles!' => \$output_roles, @@ -1023,7 +1025,8 @@ MAINTAINER field selection options: --r => include reviewer(s) if any --n => include name 'Full Name <addr\@domain.tld>' --l => include list(s) if any - --s => include subscriber only list(s) if any + --moderated => include moderated lists(s) if any (default: true) + --s => include subscriber only list(s) if any (default: false) --remove-duplicates => minimize duplicate email names/addresses --roles => show roles (status:subsystem, git-signer, list, etc...) --rolestats => show roles and statistics (commits/total_commits, %) @@ -1313,11 +1316,14 @@ sub add_categories { } else { if ($email_list) { if (!$hash_list_to{lc($list_address)}) { - $hash_list_to{lc($list_address)} = 1; if ($list_additional =~ m/moderated/) { - push(@list_to, [$list_address, - "moderated list${list_role}"]); + if ($email_moderated_list) { + $hash_list_to{lc($list_address)} = 1; + push(@list_to, [$list_address, + "moderated list${list_role}"]); + } } else { + $hash_list_to{lc($list_address)} = 1; push(@list_to, [$list_address, "open list${list_role}"]); } diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index a60e7a17f0b8..7737b2670064 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -1021,7 +1021,7 @@ static ssize_t snd_seq_write(struct file *file, const char __user *buf, { struct snd_seq_client *client = file->private_data; int written = 0, len; - int err; + int err, handled; struct snd_seq_event event; if (!(snd_seq_file_flags(file) & SNDRV_SEQ_LFLG_OUTPUT)) @@ -1034,6 +1034,8 @@ static ssize_t snd_seq_write(struct file *file, const char __user *buf, if (!client->accept_output || client->pool == NULL) return -ENXIO; + repeat: + handled = 0; /* allocate the pool now if the pool is not allocated yet */ mutex_lock(&client->ioctl_mutex); if (client->pool->size > 0 && !snd_seq_write_pool_allocated(client)) { @@ -1093,12 +1095,19 @@ static ssize_t snd_seq_write(struct file *file, const char __user *buf, 0, 0, &client->ioctl_mutex); if (err < 0) break; + handled++; __skip_event: /* Update pointers and counts */ count -= len; buf += len; written += len; + + /* let's have a coffee break if too many events are queued */ + if (++handled >= 200) { + mutex_unlock(&client->ioctl_mutex); + goto repeat; + } } out: diff --git a/sound/pci/au88x0/au88x0_a3d.c b/sound/pci/au88x0/au88x0_a3d.c index 73471037d59f..2db183f8826a 100644 --- a/sound/pci/au88x0/au88x0_a3d.c +++ b/sound/pci/au88x0/au88x0_a3d.c @@ -765,7 +765,7 @@ snd_vortex_a3d_hrtf_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { a3dsrc_t *a = kcontrol->private_data; - int changed = 1, i; + int i; int coord[6]; for (i = 0; i < 6; i++) coord[i] = ucontrol->value.integer.value[i]; @@ -774,7 +774,7 @@ snd_vortex_a3d_hrtf_put(struct snd_kcontrol *kcontrol, vortex_a3d_coord2hrtf(a->hrtf[1], coord); a3dsrc_SetHrtfTarget(a, a->hrtf[0], a->hrtf[1]); a3dsrc_SetHrtfCurrent(a, a->hrtf[0], a->hrtf[1]); - return changed; + return 1; } static int @@ -783,7 +783,7 @@ snd_vortex_a3d_itd_put(struct snd_kcontrol *kcontrol, { a3dsrc_t *a = kcontrol->private_data; int coord[6]; - int i, changed = 1; + int i; for (i = 0; i < 6; i++) coord[i] = ucontrol->value.integer.value[i]; /* Translate orientation coordinates to a3d params. */ @@ -793,7 +793,7 @@ snd_vortex_a3d_itd_put(struct snd_kcontrol *kcontrol, a3dsrc_SetItdTarget(a, a->itd[0], a->itd[1]); a3dsrc_SetItdCurrent(a, a->itd[0], a->itd[1]); a3dsrc_SetItdDline(a, a->dline); - return changed; + return 1; } static int @@ -801,7 +801,6 @@ snd_vortex_a3d_ild_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { a3dsrc_t *a = kcontrol->private_data; - int changed = 1; int l, r; /* There may be some scale tranlation needed here. */ l = ucontrol->value.integer.value[0]; @@ -810,7 +809,7 @@ snd_vortex_a3d_ild_put(struct snd_kcontrol *kcontrol, /* Left Right panning. */ a3dsrc_SetGainTarget(a, l, r); a3dsrc_SetGainCurrent(a, l, r); - return changed; + return 1; } static int @@ -818,7 +817,7 @@ snd_vortex_a3d_filter_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { a3dsrc_t *a = kcontrol->private_data; - int i, changed = 1; + int i; int params[6]; for (i = 0; i < 6; i++) params[i] = ucontrol->value.integer.value[i]; @@ -831,7 +830,7 @@ snd_vortex_a3d_filter_put(struct snd_kcontrol *kcontrol, a3dsrc_SetAtmosCurrent(a, a->filter[0], a->filter[1], a->filter[2], a->filter[3], a->filter[4]); - return changed; + return 1; } static const struct snd_kcontrol_new vortex_a3d_kcontrol = { diff --git a/sound/pci/emu10k1/emu10k1x.c b/sound/pci/emu10k1/emu10k1x.c index 67d6473ab0cd..9cf81832259c 100644 --- a/sound/pci/emu10k1/emu10k1x.c +++ b/sound/pci/emu10k1/emu10k1x.c @@ -1074,7 +1074,6 @@ static int snd_emu10k1x_shared_spdif_put(struct snd_kcontrol *kcontrol, { struct emu10k1x *emu = snd_kcontrol_chip(kcontrol); unsigned int val; - int change = 0; val = ucontrol->value.integer.value[0] ; @@ -1089,7 +1088,7 @@ static int snd_emu10k1x_shared_spdif_put(struct snd_kcontrol *kcontrol, snd_emu10k1x_ptr_write(emu, ROUTING, 0, 0x1003F); snd_emu10k1x_gpio_write(emu, 0x1080); } - return change; + return 0; } static const struct snd_kcontrol_new snd_emu10k1x_shared_spdif = diff --git a/sound/pci/hda/hda_codec.c b/sound/pci/hda/hda_codec.c index 5346631df1ec..e30e86ca6b72 100644 --- a/sound/pci/hda/hda_codec.c +++ b/sound/pci/hda/hda_codec.c @@ -2941,15 +2941,19 @@ static int hda_codec_runtime_resume(struct device *dev) #ifdef CONFIG_PM_SLEEP static int hda_codec_force_resume(struct device *dev) { + struct hda_codec *codec = dev_to_hda_codec(dev); + bool forced_resume = !codec->relaxed_resume; int ret; /* The get/put pair below enforces the runtime resume even if the * device hasn't been used at suspend time. This trick is needed to * update the jack state change during the sleep. */ - pm_runtime_get_noresume(dev); + if (forced_resume) + pm_runtime_get_noresume(dev); ret = pm_runtime_force_resume(dev); - pm_runtime_put(dev); + if (forced_resume) + pm_runtime_put(dev); return ret; } diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index 40323d91f9e4..bea7b0961080 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -2292,8 +2292,10 @@ static void generic_hdmi_free(struct hda_codec *codec) struct hdmi_spec *spec = codec->spec; int pin_idx, pcm_idx; - if (codec_has_acomp(codec)) + if (codec_has_acomp(codec)) { snd_hdac_acomp_register_notifier(&codec->bus->core, NULL); + codec->relaxed_resume = 0; + } for (pin_idx = 0; pin_idx < spec->num_pins; pin_idx++) { struct hdmi_spec_per_pin *per_pin = get_pin(spec, pin_idx); @@ -2417,7 +2419,6 @@ static void intel_haswell_fixup_connect_list(struct hda_codec *codec, } #define INTEL_GET_VENDOR_VERB 0xf81 -#define INTEL_GET_VENDOR_VERB 0xf81 #define INTEL_SET_VENDOR_VERB 0x781 #define INTEL_EN_DP12 0x02 /* enable DP 1.2 features */ #define INTEL_EN_ALL_PIN_CVTS 0x01 /* enable 2nd & 3rd pins and convertors */ @@ -2525,18 +2526,32 @@ static int intel_pin2port(void *audio_ptr, int pin_nid) return -1; } +static int intel_port2pin(struct hda_codec *codec, int port) +{ + struct hdmi_spec *spec = codec->spec; + + if (!spec->port_num) { + /* we assume only from port-B to port-D */ + if (port < 1 || port > 3) + return 0; + /* intel port is 1-based */ + return port + intel_base_nid(codec) - 1; + } + + if (port < 1 || port > spec->port_num) + return 0; + return spec->port_map[port - 1]; +} + static void intel_pin_eld_notify(void *audio_ptr, int port, int pipe) { struct hda_codec *codec = audio_ptr; int pin_nid; int dev_id = pipe; - /* we assume only from port-B to port-D */ - if (port < 1 || port > 3) + pin_nid = intel_port2pin(codec, port); + if (!pin_nid) return; - - pin_nid = port + intel_base_nid(codec) - 1; /* intel port is 1-based */ - /* skip notification during system suspend (but not in runtime PM); * the state will be updated at resume */ @@ -2566,6 +2581,8 @@ static void register_i915_notifier(struct hda_codec *codec) spec->drm_audio_ops.pin_eld_notify = intel_pin_eld_notify; snd_hdac_acomp_register_notifier(&codec->bus->core, &spec->drm_audio_ops); + /* no need for forcible resume for jack check thanks to notifier */ + codec->relaxed_resume = 1; } /* setup_stream ops override for HSW+ */ diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index f24a757f8239..de224cbea7a0 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -7657,9 +7657,12 @@ static const struct snd_hda_pin_quirk alc269_pin_fixup_tbl[] = { {0x12, 0x90a60130}, {0x17, 0x90170110}, {0x21, 0x03211020}), - SND_HDA_PIN_QUIRK(0x10ec0295, 0x1028, "Dell", ALC269_FIXUP_DELL1_MIC_NO_PRESENCE, + SND_HDA_PIN_QUIRK(0x10ec0295, 0x1028, "Dell", ALC269_FIXUP_DELL4_MIC_NO_PRESENCE, {0x14, 0x90170110}, {0x21, 0x04211020}), + SND_HDA_PIN_QUIRK(0x10ec0295, 0x1028, "Dell", ALC269_FIXUP_DELL4_MIC_NO_PRESENCE, + {0x14, 0x90170110}, + {0x21, 0x04211030}), SND_HDA_PIN_QUIRK(0x10ec0295, 0x1028, "Dell", ALC269_FIXUP_DELL1_MIC_NO_PRESENCE, ALC295_STANDARD_PINS, {0x17, 0x21014020}, @@ -8800,6 +8803,11 @@ static const struct snd_hda_pin_quirk alc662_pin_fixup_tbl[] = { {0x18, 0x01a19030}, {0x1a, 0x01813040}, {0x21, 0x01014020}), + SND_HDA_PIN_QUIRK(0x10ec0867, 0x1028, "Dell", ALC891_FIXUP_DELL_MIC_NO_PRESENCE, + {0x16, 0x01813030}, + {0x17, 0x02211010}, + {0x18, 0x01a19040}, + {0x21, 0x01014020}), SND_HDA_PIN_QUIRK(0x10ec0662, 0x1028, "Dell", ALC662_FIXUP_DELL_MIC_NO_PRESENCE, {0x14, 0x01014010}, {0x18, 0x01a19020}, diff --git a/sound/pci/lx6464es/lx6464es.c b/sound/pci/lx6464es/lx6464es.c index 1771a6dcbe18..583ca7384d83 100644 --- a/sound/pci/lx6464es/lx6464es.c +++ b/sound/pci/lx6464es/lx6464es.c @@ -253,9 +253,8 @@ exit: static int lx_pcm_close(struct snd_pcm_substream *substream) { - int err = 0; dev_dbg(substream->pcm->card->dev, "->lx_pcm_close\n"); - return err; + return 0; } static snd_pcm_uframes_t lx_pcm_stream_pointer(struct snd_pcm_substream diff --git a/sound/pci/rme9652/rme9652.c b/sound/pci/rme9652/rme9652.c index cb9818af5b41..4c851f8dcaf8 100644 --- a/sound/pci/rme9652/rme9652.c +++ b/sound/pci/rme9652/rme9652.c @@ -2158,13 +2158,12 @@ static int snd_rme9652_prepare(struct snd_pcm_substream *substream) { struct snd_rme9652 *rme9652 = snd_pcm_substream_chip(substream); unsigned long flags; - int result = 0; spin_lock_irqsave(&rme9652->lock, flags); if (!rme9652->running) rme9652_reset_hw_pointer(rme9652); spin_unlock_irqrestore(&rme9652->lock, flags); - return result; + return 0; } static const struct snd_pcm_hardware snd_rme9652_playback_subinfo = diff --git a/sound/ppc/snd_ps3.c b/sound/ppc/snd_ps3.c index 71b7fd344c58..c213eb7ca23c 100644 --- a/sound/ppc/snd_ps3.c +++ b/sound/ppc/snd_ps3.c @@ -628,7 +628,6 @@ static int snd_ps3_pcm_trigger(struct snd_pcm_substream *substream, int cmd) { struct snd_ps3_card_info *card = snd_pcm_substream_chip(substream); - int ret = 0; switch (cmd) { case SNDRV_PCM_TRIGGER_START: @@ -665,7 +664,7 @@ static int snd_ps3_pcm_trigger(struct snd_pcm_substream *substream, } - return ret; + return 0; }; /* diff --git a/sound/soc/qcom/qdsp6/q6asm.c b/sound/soc/qcom/qdsp6/q6asm.c index 4f85cb19a309..e8141a33a55e 100644 --- a/sound/soc/qcom/qdsp6/q6asm.c +++ b/sound/soc/qcom/qdsp6/q6asm.c @@ -1194,7 +1194,7 @@ EXPORT_SYMBOL_GPL(q6asm_open_read); * q6asm_write_async() - non blocking write * * @ac: audio client pointer - * @len: lenght in bytes + * @len: length in bytes * @msw_ts: timestamp msw * @lsw_ts: timestamp lsw * @wflags: flags associated with write diff --git a/tools/perf/Documentation/perf-probe.txt b/tools/perf/Documentation/perf-probe.txt index b6866a05edd2..ed3ecfa422e1 100644 --- a/tools/perf/Documentation/perf-probe.txt +++ b/tools/perf/Documentation/perf-probe.txt @@ -194,12 +194,13 @@ PROBE ARGUMENT -------------- Each probe argument follows below syntax. - [NAME=]LOCALVAR|$retval|%REG|@SYMBOL[:TYPE] + [NAME=]LOCALVAR|$retval|%REG|@SYMBOL[:TYPE][@user] 'NAME' specifies the name of this argument (optional). You can use the name of local variable, local data structure member (e.g. var->field, var.field2), local array with fixed index (e.g. array[1], var->array[0], var->pointer[2]), or kprobe-tracer argument format (e.g. $retval, %ax, etc). Note that the name of this argument will be set as the last member name if you specify a local data structure member (e.g. field2 for 'var->field1.field2'.) '$vars' and '$params' special arguments are also available for NAME, '$vars' is expanded to the local variables (including function parameters) which can access at given probe point. '$params' is expanded to only the function parameters. 'TYPE' casts the type of this argument (optional). If omitted, perf probe automatically set the type based on debuginfo (*). Currently, basic types (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal integers (x/x8/x16/x32/x64), signedness casting (u/s), "string" and bitfield are supported. (see TYPES for detail) On x86 systems %REG is always the short form of the register: for example %AX. %RAX or %EAX is not valid. +"@user" is a special attribute which means the LOCALVAR will be treated as a user-space memory. This is only valid for kprobe event. TYPES ----- diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c index 0c3b55d0617d..cd1eb73cfe83 100644 --- a/tools/perf/util/probe-event.c +++ b/tools/perf/util/probe-event.c @@ -1562,6 +1562,17 @@ static int parse_perf_probe_arg(char *str, struct perf_probe_arg *arg) str = tmp + 1; } + tmp = strchr(str, '@'); + if (tmp && tmp != str && strcmp(tmp + 1, "user")) { /* user attr */ + if (!user_access_is_supported()) { + semantic_error("ftrace does not support user access\n"); + return -EINVAL; + } + *tmp = '\0'; + arg->user_access = true; + pr_debug("user_access "); + } + tmp = strchr(str, ':'); if (tmp) { /* Type setting */ *tmp = '\0'; diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h index 05c8d571a901..96a319cd2378 100644 --- a/tools/perf/util/probe-event.h +++ b/tools/perf/util/probe-event.h @@ -37,6 +37,7 @@ struct probe_trace_point { struct probe_trace_arg_ref { struct probe_trace_arg_ref *next; /* Next reference */ long offset; /* Offset value */ + bool user_access; /* User-memory access */ }; /* kprobe-tracer and uprobe-tracer tracing argument */ @@ -82,6 +83,7 @@ struct perf_probe_arg { char *var; /* Variable name */ char *type; /* Type name */ struct perf_probe_arg_field *field; /* Structure fields */ + bool user_access; /* User-memory access */ }; /* Perf probe probing event (point + arg) */ diff --git a/tools/perf/util/probe-file.c b/tools/perf/util/probe-file.c index c2998f90b23c..5b4d49382932 100644 --- a/tools/perf/util/probe-file.c +++ b/tools/perf/util/probe-file.c @@ -1005,6 +1005,7 @@ enum ftrace_readme { FTRACE_README_PROBE_TYPE_X = 0, FTRACE_README_KRETPROBE_OFFSET, FTRACE_README_UPROBE_REF_CTR, + FTRACE_README_USER_ACCESS, FTRACE_README_END, }; @@ -1017,6 +1018,7 @@ static struct { DEFINE_TYPE(FTRACE_README_PROBE_TYPE_X, "*type: * x8/16/32/64,*"), DEFINE_TYPE(FTRACE_README_KRETPROBE_OFFSET, "*place (kretprobe): *"), DEFINE_TYPE(FTRACE_README_UPROBE_REF_CTR, "*ref_ctr_offset*"), + DEFINE_TYPE(FTRACE_README_USER_ACCESS, "*[u]<offset>*"), }; static bool scan_ftrace_readme(enum ftrace_readme type) @@ -1077,3 +1079,8 @@ bool uprobe_ref_ctr_is_supported(void) { return scan_ftrace_readme(FTRACE_README_UPROBE_REF_CTR); } + +bool user_access_is_supported(void) +{ + return scan_ftrace_readme(FTRACE_README_USER_ACCESS); +} diff --git a/tools/perf/util/probe-file.h b/tools/perf/util/probe-file.h index 2a249182f2a6..986c1c94f64f 100644 --- a/tools/perf/util/probe-file.h +++ b/tools/perf/util/probe-file.h @@ -70,6 +70,7 @@ int probe_cache__show_all_caches(struct strfilter *filter); bool probe_type_is_available(enum probe_type type); bool kretprobe_offset_is_supported(void); bool uprobe_ref_ctr_is_supported(void); +bool user_access_is_supported(void); #else /* ! HAVE_LIBELF_SUPPORT */ static inline struct probe_cache *probe_cache__new(const char *tgt __maybe_unused, struct nsinfo *nsi __maybe_unused) { diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c index 7d8c99734928..025fc4491993 100644 --- a/tools/perf/util/probe-finder.c +++ b/tools/perf/util/probe-finder.c @@ -280,7 +280,7 @@ static_var: static int convert_variable_type(Dwarf_Die *vr_die, struct probe_trace_arg *tvar, - const char *cast) + const char *cast, bool user_access) { struct probe_trace_arg_ref **ref_ptr = &tvar->ref; Dwarf_Die type; @@ -320,7 +320,8 @@ static int convert_variable_type(Dwarf_Die *vr_die, pr_debug("%s type is %s.\n", dwarf_diename(vr_die), dwarf_diename(&type)); - if (cast && strcmp(cast, "string") == 0) { /* String type */ + if (cast && (!strcmp(cast, "string") || !strcmp(cast, "ustring"))) { + /* String type */ ret = dwarf_tag(&type); if (ret != DW_TAG_pointer_type && ret != DW_TAG_array_type) { @@ -343,6 +344,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, pr_warning("Out of memory error\n"); return -ENOMEM; } + (*ref_ptr)->user_access = user_access; } if (!die_compare_name(&type, "char") && !die_compare_name(&type, "unsigned char")) { @@ -397,7 +399,7 @@ formatted: static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, struct perf_probe_arg_field *field, struct probe_trace_arg_ref **ref_ptr, - Dwarf_Die *die_mem) + Dwarf_Die *die_mem, bool user_access) { struct probe_trace_arg_ref *ref = *ref_ptr; Dwarf_Die type; @@ -434,6 +436,7 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, *ref_ptr = ref; } ref->offset += dwarf_bytesize(&type) * field->index; + ref->user_access = user_access; goto next; } else if (tag == DW_TAG_pointer_type) { /* Check the pointer and dereference */ @@ -505,17 +508,18 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, } } ref->offset += (long)offs; + ref->user_access = user_access; /* If this member is unnamed, we need to reuse this field */ if (!dwarf_diename(die_mem)) return convert_variable_fields(die_mem, varname, field, - &ref, die_mem); + &ref, die_mem, user_access); next: /* Converting next field */ if (field->next) return convert_variable_fields(die_mem, field->name, - field->next, &ref, die_mem); + field->next, &ref, die_mem, user_access); else return 0; } @@ -541,11 +545,12 @@ static int convert_variable(Dwarf_Die *vr_die, struct probe_finder *pf) else if (ret == 0 && pf->pvar->field) { ret = convert_variable_fields(vr_die, pf->pvar->var, pf->pvar->field, &pf->tvar->ref, - &die_mem); + &die_mem, pf->pvar->user_access); vr_die = &die_mem; } if (ret == 0) - ret = convert_variable_type(vr_die, pf->tvar, pf->pvar->type); + ret = convert_variable_type(vr_die, pf->tvar, pf->pvar->type, + pf->pvar->user_access); /* *expr will be cached in libdw. Don't free it. */ return ret; } diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index 6d5e9e87c4b7..063ecb290a5a 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -23,9 +23,15 @@ echo " If <dir> is -, all logs output in console only" exit $1 } +# default error +err_ret=1 + +# kselftest skip code is 4 +err_skip=4 + errexit() { # message echo "Error: $1" 1>&2 - exit 1 + exit $err_ret } # Ensuring user privilege @@ -116,11 +122,31 @@ parse_opts() { # opts } # Parameters -DEBUGFS_DIR=`grep debugfs /proc/mounts | cut -f2 -d' ' | head -1` -if [ -z "$DEBUGFS_DIR" ]; then - TRACING_DIR=`grep tracefs /proc/mounts | cut -f2 -d' ' | head -1` -else - TRACING_DIR=$DEBUGFS_DIR/tracing +TRACING_DIR=`grep tracefs /proc/mounts | cut -f2 -d' ' | head -1` +if [ -z "$TRACING_DIR" ]; then + DEBUGFS_DIR=`grep debugfs /proc/mounts | cut -f2 -d' ' | head -1` + if [ -z "$DEBUGFS_DIR" ]; then + # If tracefs exists, then so does /sys/kernel/tracing + if [ -d "/sys/kernel/tracing" ]; then + mount -t tracefs nodev /sys/kernel/tracing || + errexit "Failed to mount /sys/kernel/tracing" + TRACING_DIR="/sys/kernel/tracing" + # If debugfs exists, then so does /sys/kernel/debug + elif [ -d "/sys/kernel/debug" ]; then + mount -t debugfs nodev /sys/kernel/debug || + errexit "Failed to mount /sys/kernel/debug" + TRACING_DIR="/sys/kernel/debug/tracing" + else + err_ret=$err_skip + errexit "debugfs and tracefs are not configured in this kernel" + fi + else + TRACING_DIR="$DEBUGFS_DIR/tracing" + fi +fi +if [ ! -d "$TRACING_DIR" ]; then + err_ret=$err_skip + errexit "ftrace is not configured in this kernel" fi TOP_DIR=`absdir $0` diff --git a/tools/testing/selftests/ftrace/test.d/functions b/tools/testing/selftests/ftrace/test.d/functions index 779ec11f61bd..1d96c5f7e402 100644 --- a/tools/testing/selftests/ftrace/test.d/functions +++ b/tools/testing/selftests/ftrace/test.d/functions @@ -91,8 +91,8 @@ initialize_ftrace() { # Reset ftrace to initial-state reset_events_filter reset_ftrace_filter disable_events - echo > set_event_pid # event tracer is always on - echo > set_ftrace_pid + [ -f set_event_pid ] && echo > set_event_pid + [ -f set_ftrace_pid ] && echo > set_ftrace_pid [ -f set_ftrace_filter ] && echo | tee set_ftrace_* [ -f set_graph_function ] && echo | tee set_graph_* [ -f stack_trace_filter ] && echo > stack_trace_filter diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_user.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_user.tc new file mode 100644 index 000000000000..0f60087583d8 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_user.tc @@ -0,0 +1,32 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Kprobe event user-memory access + +[ -f kprobe_events ] || exit_unsupported # this is configurable + +grep -q '\$arg<N>' README || exit_unresolved # depends on arch +grep -A10 "fetcharg:" README | grep -q 'ustring' || exit_unsupported +grep -A10 "fetcharg:" README | grep -q '\[u\]<offset>' || exit_unsupported + +:;: "user-memory access syntax and ustring working on user memory";: +echo 'p:myevent do_sys_open path=+0($arg2):ustring path2=+u0($arg2):string' \ + > kprobe_events + +grep myevent kprobe_events | \ + grep -q 'path=+0($arg2):ustring path2=+u0($arg2):string' +echo 1 > events/kprobes/myevent/enable +echo > /dev/null +echo 0 > events/kprobes/myevent/enable + +grep myevent trace | grep -q 'path="/dev/null" path2="/dev/null"' + +:;: "user-memory access syntax and ustring not working with kernel memory";: +echo 'p:myevent vfs_symlink path=+0($arg3):ustring path2=+u0($arg3):string' \ + > kprobe_events +echo 1 > events/kprobes/myevent/enable +ln -s foo $TMPDIR/bar +echo 0 > events/kprobes/myevent/enable + +grep myevent trace | grep -q 'path=(fault) path2=(fault)' + +exit 0 diff --git a/tools/testing/selftests/proc/.gitignore b/tools/testing/selftests/proc/.gitignore index 444ad39d3700..66fab4c58ed4 100644 --- a/tools/testing/selftests/proc/.gitignore +++ b/tools/testing/selftests/proc/.gitignore @@ -12,4 +12,5 @@ /read /self /setns-dcache +/setns-sysvipc /thread-self diff --git a/tools/testing/selftests/proc/Makefile b/tools/testing/selftests/proc/Makefile index 9f09fcd09ea3..a8ed0f684829 100644 --- a/tools/testing/selftests/proc/Makefile +++ b/tools/testing/selftests/proc/Makefile @@ -17,6 +17,7 @@ TEST_GEN_PROGS += proc-uptime-002 TEST_GEN_PROGS += read TEST_GEN_PROGS += self TEST_GEN_PROGS += setns-dcache +TEST_GEN_PROGS += setns-sysvipc TEST_GEN_PROGS += thread-self include ../lib.mk diff --git a/tools/testing/selftests/proc/proc-pid-vm.c b/tools/testing/selftests/proc/proc-pid-vm.c index 853aa164a401..18a3bde8bc96 100644 --- a/tools/testing/selftests/proc/proc-pid-vm.c +++ b/tools/testing/selftests/proc/proc-pid-vm.c @@ -215,6 +215,11 @@ static const char str_vsyscall[] = "ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n"; #ifdef __x86_64__ +static void sigaction_SIGSEGV(int _, siginfo_t *__, void *___) +{ + _exit(1); +} + /* * vsyscall page can't be unmapped, probe it with memory load. */ @@ -231,11 +236,19 @@ static void vsyscall(void) if (pid == 0) { struct rlimit rlim = {0, 0}; (void)setrlimit(RLIMIT_CORE, &rlim); + + /* Hide "segfault at ffffffffff600000" messages. */ + struct sigaction act; + memset(&act, 0, sizeof(struct sigaction)); + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = sigaction_SIGSEGV; + (void)sigaction(SIGSEGV, &act, NULL); + *(volatile int *)0xffffffffff600000UL; exit(0); } - wait(&wstatus); - if (WIFEXITED(wstatus)) { + waitpid(pid, &wstatus, 0); + if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0) { g_vsyscall = true; } } diff --git a/tools/testing/selftests/proc/setns-sysvipc.c b/tools/testing/selftests/proc/setns-sysvipc.c new file mode 100644 index 000000000000..903890c5e587 --- /dev/null +++ b/tools/testing/selftests/proc/setns-sysvipc.c @@ -0,0 +1,133 @@ +/* + * Copyright © 2019 Alexey Dobriyan <adobriyan@gmail.com> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +/* + * Test that setns(CLONE_NEWIPC) points to new /proc/sysvipc content even + * if old one is in dcache. + */ +#undef NDEBUG +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <sched.h> +#include <signal.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <sys/ipc.h> +#include <sys/shm.h> + +static pid_t pid = -1; + +static void f(void) +{ + if (pid > 0) { + kill(pid, SIGTERM); + } +} + +int main(void) +{ + int fd[2]; + char _ = 0; + int nsfd; + + atexit(f); + + /* Check for priviledges and syscall availability straight away. */ + if (unshare(CLONE_NEWIPC) == -1) { + if (errno == ENOSYS || errno == EPERM) { + return 4; + } + return 1; + } + /* Distinguisher between two otherwise empty IPC namespaces. */ + if (shmget(IPC_PRIVATE, 1, IPC_CREAT) == -1) { + return 1; + } + + if (pipe(fd) == -1) { + return 1; + } + + pid = fork(); + if (pid == -1) { + return 1; + } + + if (pid == 0) { + if (unshare(CLONE_NEWIPC) == -1) { + return 1; + } + + if (write(fd[1], &_, 1) != 1) { + return 1; + } + + pause(); + + return 0; + } + + if (read(fd[0], &_, 1) != 1) { + return 1; + } + + { + char buf[64]; + snprintf(buf, sizeof(buf), "/proc/%u/ns/ipc", pid); + nsfd = open(buf, O_RDONLY); + if (nsfd == -1) { + return 1; + } + } + + /* Reliably pin dentry into dcache. */ + (void)open("/proc/sysvipc/shm", O_RDONLY); + + if (setns(nsfd, CLONE_NEWIPC) == -1) { + return 1; + } + + kill(pid, SIGTERM); + pid = 0; + + { + char buf[4096]; + ssize_t rv; + int fd; + + fd = open("/proc/sysvipc/shm", O_RDONLY); + if (fd == -1) { + return 1; + } + +#define S32 " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime rss swap\n" +#define S64 " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime rss swap\n" + rv = read(fd, buf, sizeof(buf)); + if (rv == strlen(S32)) { + assert(memcmp(buf, S32, strlen(S32)) == 0); + } else if (rv == strlen(S64)) { + assert(memcmp(buf, S64, strlen(S64)) == 0); + } else { + assert(0); + } + } + + return 0; +} diff --git a/tools/testing/selftests/ptrace/.gitignore b/tools/testing/selftests/ptrace/.gitignore index b3e59d41fd82..cfcc49a7def7 100644 --- a/tools/testing/selftests/ptrace/.gitignore +++ b/tools/testing/selftests/ptrace/.gitignore @@ -1 +1,2 @@ +get_syscall_info peeksiginfo diff --git a/tools/testing/selftests/ptrace/Makefile b/tools/testing/selftests/ptrace/Makefile index cb21c76a18ca..c0b7f89f0930 100644 --- a/tools/testing/selftests/ptrace/Makefile +++ b/tools/testing/selftests/ptrace/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS += -iquote../../../../include/uapi -Wall -TEST_GEN_PROGS := peeksiginfo +TEST_GEN_PROGS := get_syscall_info peeksiginfo include ../lib.mk diff --git a/tools/testing/selftests/ptrace/get_syscall_info.c b/tools/testing/selftests/ptrace/get_syscall_info.c new file mode 100644 index 000000000000..5bcd1c7b5be6 --- /dev/null +++ b/tools/testing/selftests/ptrace/get_syscall_info.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2018 Dmitry V. Levin <ldv@altlinux.org> + * All rights reserved. + * + * Check whether PTRACE_GET_SYSCALL_INFO semantics implemented in the kernel + * matches userspace expectations. + */ + +#include "../kselftest_harness.h" +#include <err.h> +#include <signal.h> +#include <asm/unistd.h> +#include "linux/ptrace.h" + +static int +kill_tracee(pid_t pid) +{ + if (!pid) + return 0; + + int saved_errno = errno; + + int rc = kill(pid, SIGKILL); + + errno = saved_errno; + return rc; +} + +static long +sys_ptrace(int request, pid_t pid, unsigned long addr, unsigned long data) +{ + return syscall(__NR_ptrace, request, pid, addr, data); +} + +#define LOG_KILL_TRACEE(fmt, ...) \ + do { \ + kill_tracee(pid); \ + TH_LOG("wait #%d: " fmt, \ + ptrace_stop, ##__VA_ARGS__); \ + } while (0) + +TEST(get_syscall_info) +{ + static const unsigned long args[][7] = { + /* a sequence of architecture-agnostic syscalls */ + { + __NR_chdir, + (unsigned long) "", + 0xbad1fed1, + 0xbad2fed2, + 0xbad3fed3, + 0xbad4fed4, + 0xbad5fed5 + }, + { + __NR_gettid, + 0xcaf0bea0, + 0xcaf1bea1, + 0xcaf2bea2, + 0xcaf3bea3, + 0xcaf4bea4, + 0xcaf5bea5 + }, + { + __NR_exit_group, + 0, + 0xfac1c0d1, + 0xfac2c0d2, + 0xfac3c0d3, + 0xfac4c0d4, + 0xfac5c0d5 + } + }; + const unsigned long *exp_args; + + pid_t pid = fork(); + + ASSERT_LE(0, pid) { + TH_LOG("fork: %m"); + } + + if (pid == 0) { + /* get the pid before PTRACE_TRACEME */ + pid = getpid(); + ASSERT_EQ(0, sys_ptrace(PTRACE_TRACEME, 0, 0, 0)) { + TH_LOG("PTRACE_TRACEME: %m"); + } + ASSERT_EQ(0, kill(pid, SIGSTOP)) { + /* cannot happen */ + TH_LOG("kill SIGSTOP: %m"); + } + for (unsigned int i = 0; i < ARRAY_SIZE(args); ++i) { + syscall(args[i][0], + args[i][1], args[i][2], args[i][3], + args[i][4], args[i][5], args[i][6]); + } + /* unreachable */ + _exit(1); + } + + const struct { + unsigned int is_error; + int rval; + } *exp_param, exit_param[] = { + { 1, -ENOENT }, /* chdir */ + { 0, pid } /* gettid */ + }; + + unsigned int ptrace_stop; + + for (ptrace_stop = 0; ; ++ptrace_stop) { + struct ptrace_syscall_info info = { + .op = 0xff /* invalid PTRACE_SYSCALL_INFO_* op */ + }; + const size_t size = sizeof(info); + const int expected_none_size = + (void *) &info.entry - (void *) &info; + const int expected_entry_size = + (void *) &info.entry.args[6] - (void *) &info; + const int expected_exit_size = + (void *) (&info.exit.is_error + 1) - + (void *) &info; + int status; + long rc; + + ASSERT_EQ(pid, wait(&status)) { + /* cannot happen */ + LOG_KILL_TRACEE("wait: %m"); + } + if (WIFEXITED(status)) { + pid = 0; /* the tracee is no more */ + ASSERT_EQ(0, WEXITSTATUS(status)); + break; + } + ASSERT_FALSE(WIFSIGNALED(status)) { + pid = 0; /* the tracee is no more */ + LOG_KILL_TRACEE("unexpected signal %u", + WTERMSIG(status)); + } + ASSERT_TRUE(WIFSTOPPED(status)) { + /* cannot happen */ + LOG_KILL_TRACEE("unexpected wait status %#x", status); + } + + switch (WSTOPSIG(status)) { + case SIGSTOP: + ASSERT_EQ(0, ptrace_stop) { + LOG_KILL_TRACEE("unexpected signal stop"); + } + ASSERT_EQ(0, sys_ptrace(PTRACE_SETOPTIONS, pid, 0, + PTRACE_O_TRACESYSGOOD)) { + LOG_KILL_TRACEE("PTRACE_SETOPTIONS: %m"); + } + ASSERT_LT(0, (rc = sys_ptrace(PTRACE_GET_SYSCALL_INFO, + pid, size, + (unsigned long) &info))) { + LOG_KILL_TRACEE("PTRACE_GET_SYSCALL_INFO: %m"); + } + ASSERT_EQ(expected_none_size, rc) { + LOG_KILL_TRACEE("signal stop mismatch"); + } + ASSERT_EQ(PTRACE_SYSCALL_INFO_NONE, info.op) { + LOG_KILL_TRACEE("signal stop mismatch"); + } + ASSERT_TRUE(info.arch) { + LOG_KILL_TRACEE("signal stop mismatch"); + } + ASSERT_TRUE(info.instruction_pointer) { + LOG_KILL_TRACEE("signal stop mismatch"); + } + ASSERT_TRUE(info.stack_pointer) { + LOG_KILL_TRACEE("signal stop mismatch"); + } + break; + + case SIGTRAP | 0x80: + ASSERT_LT(0, (rc = sys_ptrace(PTRACE_GET_SYSCALL_INFO, + pid, size, + (unsigned long) &info))) { + LOG_KILL_TRACEE("PTRACE_GET_SYSCALL_INFO: %m"); + } + switch (ptrace_stop) { + case 1: /* entering chdir */ + case 3: /* entering gettid */ + case 5: /* entering exit_group */ + exp_args = args[ptrace_stop / 2]; + ASSERT_EQ(expected_entry_size, rc) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(PTRACE_SYSCALL_INFO_ENTRY, info.op) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_TRUE(info.arch) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_TRUE(info.instruction_pointer) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_TRUE(info.stack_pointer) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(exp_args[0], info.entry.nr) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(exp_args[1], info.entry.args[0]) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(exp_args[2], info.entry.args[1]) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(exp_args[3], info.entry.args[2]) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(exp_args[4], info.entry.args[3]) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(exp_args[5], info.entry.args[4]) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + ASSERT_EQ(exp_args[6], info.entry.args[5]) { + LOG_KILL_TRACEE("entry stop mismatch"); + } + break; + case 2: /* exiting chdir */ + case 4: /* exiting gettid */ + exp_param = &exit_param[ptrace_stop / 2 - 1]; + ASSERT_EQ(expected_exit_size, rc) { + LOG_KILL_TRACEE("exit stop mismatch"); + } + ASSERT_EQ(PTRACE_SYSCALL_INFO_EXIT, info.op) { + LOG_KILL_TRACEE("exit stop mismatch"); + } + ASSERT_TRUE(info.arch) { + LOG_KILL_TRACEE("exit stop mismatch"); + } + ASSERT_TRUE(info.instruction_pointer) { + LOG_KILL_TRACEE("exit stop mismatch"); + } + ASSERT_TRUE(info.stack_pointer) { + LOG_KILL_TRACEE("exit stop mismatch"); + } + ASSERT_EQ(exp_param->is_error, + info.exit.is_error) { + LOG_KILL_TRACEE("exit stop mismatch"); + } + ASSERT_EQ(exp_param->rval, info.exit.rval) { + LOG_KILL_TRACEE("exit stop mismatch"); + } + break; + default: + LOG_KILL_TRACEE("unexpected syscall stop"); + abort(); + } + break; + + default: + LOG_KILL_TRACEE("unexpected stop signal %#x", + WSTOPSIG(status)); + abort(); + } + + ASSERT_EQ(0, sys_ptrace(PTRACE_SYSCALL, pid, 0, 0)) { + LOG_KILL_TRACEE("PTRACE_SYSCALL: %m"); + } + } + + ASSERT_EQ(ARRAY_SIZE(args) * 2, ptrace_stop); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index dc66fe852768..6ef7f16c4cf5 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -1775,13 +1775,18 @@ void tracer_ptrace(struct __test_metadata *_metadata, pid_t tracee, unsigned long msg; static bool entry; - /* Make sure we got an empty message. */ + /* + * The traditional way to tell PTRACE_SYSCALL entry/exit + * is by counting. + */ + entry = !entry; + + /* Make sure we got an appropriate message. */ ret = ptrace(PTRACE_GETEVENTMSG, tracee, NULL, &msg); EXPECT_EQ(0, ret); - EXPECT_EQ(0, msg); + EXPECT_EQ(entry ? PTRACE_EVENTMSG_SYSCALL_ENTRY + : PTRACE_EVENTMSG_SYSCALL_EXIT, msg); - /* The only way to tell PTRACE_SYSCALL entry/exit is by counting. */ - entry = !entry; if (!entry) return; |