diff -ruw linux-4.4.115/arch/arm/boot/dts/include/dt-bindings/input/linux-event-codes.h linux-4.4.115-fbx/arch/arm/boot/dts/include/dt-bindings/input/linux-event-codes.h --- linux-4.4.115/arch/arm/boot/dts/include/dt-bindings/input/linux-event-codes.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm/boot/dts/include/dt-bindings/input/linux-event-codes.h 2019-10-29 09:26:25.545221791 +0100 @@ -611,6 +611,37 @@ #define KEY_KBDINPUTASSIST_ACCEPT 0x264 #define KEY_KBDINPUTASSIST_CANCEL 0x265 +/* Diagonal movement keys */ +#define KEY_RIGHT_UP 0x266 +#define KEY_RIGHT_DOWN 0x267 +#define KEY_LEFT_UP 0x268 +#define KEY_LEFT_DOWN 0x269 + +#define KEY_ROOT_MENU 0x26a /* Show Device's Root Menu */ +/* Show Top Menu of the Media (e.g. DVD) */ +#define KEY_MEDIA_TOP_MENU 0x26b +#define KEY_NUMERIC_11 0x26c +#define KEY_NUMERIC_12 0x26d +/* + * Toggle Audio Description: refers to an audio service that helps blind and + * visually impaired consumers understand the action in a program. Note: in + * some countries this is referred to as "Video Description". + */ +#define KEY_AUDIO_DESC 0x26e +#define KEY_3D_MODE 0x26f +#define KEY_NEXT_FAVORITE 0x270 +#define KEY_STOP_RECORD 0x271 +#define KEY_PAUSE_RECORD 0x272 +#define KEY_VOD 0x273 /* Video on Demand */ +#define KEY_UNMUTE 0x274 +#define KEY_FASTREVERSE 0x275 +#define KEY_SLOWREVERSE 0x276 +/* + * Control a data application associated with the currently viewed channel, + * e.g. teletext or data broadcast application (MHEG, MHP, HbbTV, etc.) + */ +#define KEY_DATA 0x275 + #define BTN_TRIGGER_HAPPY 0x2c0 #define BTN_TRIGGER_HAPPY1 0x2c0 #define BTN_TRIGGER_HAPPY2 0x2c1 @@ -653,6 +684,18 @@ #define BTN_TRIGGER_HAPPY39 0x2e6 #define BTN_TRIGGER_HAPPY40 0x2e7 +#define KEY_APP_TV 0x2f1 +#define KEY_APP_REPLAY 0x2f2 +#define KEY_APP_VIDEOCLUB 0x2f3 +#define KEY_APP_WHATSON 0x2f4 +#define KEY_APP_RECORDS 0x2f5 +#define KEY_APP_MEDIA 0x2f6 +#define KEY_APP_YOUTUBE 0x2f7 +#define KEY_APP_RADIOS 0x2f8 +#define KEY_APP_CANALVOD 0x2f9 +#define KEY_APP_PIP 0x2fa +#define KEY_APP_NETFLIX 0x2fb + /* We avoid low common keys in module aliases so they don't get huge. */ #define KEY_MIN_INTERESTING KEY_MUTE #define KEY_MAX 0x2ff @@ -749,7 +792,11 @@ #define SW_ROTATE_LOCK 0x0c /* set = rotate locked/disabled */ #define SW_LINEIN_INSERT 0x0d /* set = inserted */ #define SW_MUTE_DEVICE 0x0e /* set = device disabled */ -#define SW_MAX 0x0f +#define SW_HPHL_OVERCURRENT 0x0f /* set = over current on left hph */ +#define SW_HPHR_OVERCURRENT 0x10 /* set = over current on right hph */ +#define SW_MICROPHONE2_INSERT 0x11 /* set = inserted */ +#define SW_UNSUPPORT_INSERT 0x12 /* set = unsupported device inserted */ +#define SW_MAX 0x20 #define SW_CNT (SW_MAX+1) /* diff -ruw linux-4.4.115/arch/arm64/boot/dts/include/dt-bindings/input/linux-event-codes.h linux-4.4.115-fbx/arch/arm64/boot/dts/include/dt-bindings/input/linux-event-codes.h --- linux-4.4.115/arch/arm64/boot/dts/include/dt-bindings/input/linux-event-codes.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/boot/dts/include/dt-bindings/input/linux-event-codes.h 2019-10-29 09:26:25.545221791 +0100 @@ -611,6 +611,37 @@ #define KEY_KBDINPUTASSIST_ACCEPT 0x264 #define KEY_KBDINPUTASSIST_CANCEL 0x265 +/* Diagonal movement keys */ +#define KEY_RIGHT_UP 0x266 +#define KEY_RIGHT_DOWN 0x267 +#define KEY_LEFT_UP 0x268 +#define KEY_LEFT_DOWN 0x269 + +#define KEY_ROOT_MENU 0x26a /* Show Device's Root Menu */ +/* Show Top Menu of the Media (e.g. DVD) */ +#define KEY_MEDIA_TOP_MENU 0x26b +#define KEY_NUMERIC_11 0x26c +#define KEY_NUMERIC_12 0x26d +/* + * Toggle Audio Description: refers to an audio service that helps blind and + * visually impaired consumers understand the action in a program. Note: in + * some countries this is referred to as "Video Description". + */ +#define KEY_AUDIO_DESC 0x26e +#define KEY_3D_MODE 0x26f +#define KEY_NEXT_FAVORITE 0x270 +#define KEY_STOP_RECORD 0x271 +#define KEY_PAUSE_RECORD 0x272 +#define KEY_VOD 0x273 /* Video on Demand */ +#define KEY_UNMUTE 0x274 +#define KEY_FASTREVERSE 0x275 +#define KEY_SLOWREVERSE 0x276 +/* + * Control a data application associated with the currently viewed channel, + * e.g. teletext or data broadcast application (MHEG, MHP, HbbTV, etc.) + */ +#define KEY_DATA 0x275 + #define BTN_TRIGGER_HAPPY 0x2c0 #define BTN_TRIGGER_HAPPY1 0x2c0 #define BTN_TRIGGER_HAPPY2 0x2c1 @@ -653,6 +684,18 @@ #define BTN_TRIGGER_HAPPY39 0x2e6 #define BTN_TRIGGER_HAPPY40 0x2e7 +#define KEY_APP_TV 0x2f1 +#define KEY_APP_REPLAY 0x2f2 +#define KEY_APP_VIDEOCLUB 0x2f3 +#define KEY_APP_WHATSON 0x2f4 +#define KEY_APP_RECORDS 0x2f5 +#define KEY_APP_MEDIA 0x2f6 +#define KEY_APP_YOUTUBE 0x2f7 +#define KEY_APP_RADIOS 0x2f8 +#define KEY_APP_CANALVOD 0x2f9 +#define KEY_APP_PIP 0x2fa +#define KEY_APP_NETFLIX 0x2fb + /* We avoid low common keys in module aliases so they don't get huge. */ #define KEY_MIN_INTERESTING KEY_MUTE #define KEY_MAX 0x2ff @@ -749,7 +792,11 @@ #define SW_ROTATE_LOCK 0x0c /* set = rotate locked/disabled */ #define SW_LINEIN_INSERT 0x0d /* set = inserted */ #define SW_MUTE_DEVICE 0x0e /* set = device disabled */ -#define SW_MAX 0x0f +#define SW_HPHL_OVERCURRENT 0x0f /* set = over current on left hph */ +#define SW_HPHR_OVERCURRENT 0x10 /* set = over current on right hph */ +#define SW_MICROPHONE2_INSERT 0x11 /* set = inserted */ +#define SW_UNSUPPORT_INSERT 0x12 /* set = unsupported device inserted */ +#define SW_MAX 0x20 #define SW_CNT (SW_MAX+1) /* diff -ruw linux-4.4.115/arch/arm64/boot/dts/Makefile linux-4.4.115-fbx/arch/arm64/boot/dts/Makefile --- linux-4.4.115/arch/arm64/boot/dts/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/boot/dts/Makefile 2019-01-22 16:16:21.523228476 +0100 @@ -21,3 +21,4 @@ dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(foreach d,$(dts-dirs), $(wildcard $(dtstree)/$(d)/*.dts))) always := $(dtb-y) +targets += dtbs diff -ruw linux-4.4.115/arch/arm64/boot/dts/qcom/Makefile linux-4.4.115-fbx/arch/arm64/boot/dts/qcom/Makefile --- linux-4.4.115/arch/arm64/boot/dts/qcom/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/boot/dts/qcom/Makefile 2019-10-29 09:26:22.901195917 +0100 @@ -1,5 +1,263 @@ -dtb-$(CONFIG_ARCH_QCOM) += apq8016-sbc.dtb msm8916-mtp.dtb +dtb-$(CONFIG_ARCH_MSM8996) += msm8996-v2-pmi8994-cdp.dtb \ + msm8996-v2-pmi8994-mtp.dtb \ + msm8996-v2-pmi8994-pmk8001-cdp.dtb \ + msm8996-v2-pmi8994-pmk8001-mtp.dtb \ + msm8996-v2-pmi8994-pm8004-cdp.dtb \ + msm8996-v2-pmi8994-pm8004-mtp.dtb \ + msm8996-v2-pmi8994-pm8004-pmk8001-cdp.dtb \ + msm8996-v2-pmi8994-pm8004-pmk8001-mtp.dtb \ + msm8996-v2-fluid.dtb \ + msm8996-v2-liquid.dtb \ + msm8996-v2-dtp.dtb \ + msm8996-v3-auto-cdp.dtb \ + msm8996-v3-auto-adp.dtb \ + msm8996-v3-pmi8994-cdp.dtb \ + msm8996-v3-pmi8994-mtp.dtb \ + msm8996-v3-pmi8994-pmk8001-cdp.dtb \ + msm8996-v3-pmi8994-pmk8001-mtp.dtb \ + msm8996-v3-pmi8994-pm8004-cdp.dtb \ + msm8996-v3-pmi8994-pm8004-mtp.dtb \ + msm8996-v3-pmi8994-pm8004-pmk8001-cdp.dtb \ + msm8996-v3-pmi8994-pm8004-pmk8001-mtp.dtb \ + msm8996-v3-pmi8996-cdp.dtb \ + msm8996-v3-pmi8996-mtp.dtb \ + msm8996-v3-pmi8996-pmk8001-cdp.dtb \ + msm8996-v3-pmi8996-pmk8001-mtp.dtb \ + msm8996-v3-fluid.dtb \ + msm8996-v3-liquid.dtb \ + msm8996-v3-dtp.dtb \ + msm8996-v3-pm8004-mmxf-adp.dtb \ + msm8996-v3-pm8004-agave-adp.dtb \ + msm8996-v3-pm8004-agave-adp-lite.dtb \ + msm8996pro-auto-adp.dtb \ + msm8996pro-auto-adp-lite.dtb \ + msm8996pro-auto-cdp.dtb \ + msm8996pro-auto-cv2x.dtb \ + msm8996pro-pmi8994-cdp.dtb \ + msm8996pro-pmi8994-mtp.dtb \ + msm8996pro-pmi8994-pmk8001-cdp.dtb \ + msm8996pro-pmi8994-pmk8001-mtp.dtb \ + msm8996pro-pmi8994-pm8004-cdp.dtb \ + msm8996pro-pmi8994-pm8004-mtp.dtb \ + msm8996pro-pmi8994-pm8004-pmk8001-cdp.dtb \ + msm8996pro-pmi8994-pm8004-pmk8001-mtp.dtb \ + msm8996pro-pmi8996-cdp.dtb \ + msm8996pro-pmi8996-mtp.dtb \ + msm8996pro-pmi8996-pmk8001-cdp.dtb \ + msm8996pro-pmi8996-pmk8001-mtp.dtb \ + msm8996pro-v1.1-auto-cdp.dtb \ + msm8996pro-v1.1-pmi8994-cdp.dtb \ + msm8996pro-v1.1-pmi8994-mtp.dtb \ + msm8996pro-v1.1-pmi8994-pmk8001-cdp.dtb \ + msm8996pro-v1.1-pmi8994-pmk8001-mtp.dtb \ + msm8996pro-v1.1-pmi8994-pm8004-cdp.dtb \ + msm8996pro-v1.1-pmi8994-pm8004-mtp.dtb \ + msm8996pro-v1.1-pmi8994-pm8004-pmk8001-cdp.dtb \ + msm8996pro-v1.1-pmi8994-pm8004-pmk8001-mtp.dtb \ + msm8996pro-v1.1-pmi8996-cdp.dtb \ + msm8996pro-v1.1-pmi8996-mtp.dtb \ + msm8996pro-v1.1-pmi8996-pmk8001-cdp.dtb \ + msm8996pro-v1.1-pmi8996-pmk8001-mtp.dtb \ + apq8096pro-auto-cdp.dtb \ + apq8096pro-v1.1-auto-adp.dtb \ + apq8096pro-v1.1-auto-adp-lite.dtb \ + apq8096pro-liquid.dtb \ + apq8096pro-v1.1-auto-cdp.dtb \ + msm8996-v3.0-pmi8994-cdp.dtb \ + msm8996-v3.0-pmi8994-mtp.dtb \ + msm8996-v3.0-pmi8994-pm8004-cdp.dtb \ + msm8996-v3.0-pmi8994-pm8004-mtp.dtb \ + msm8996-v3.0-pmi8994-pm8004-pmk8001-cdp.dtb \ + msm8996-v3.0-pmi8994-pmk8001-cdp.dtb \ + msm8996-v3.0-pmi8996-cdp.dtb \ + msm8996-v3.0-pmi8996-mtp.dtb \ + msm8996-v3.0-fluid.dtb \ + msm8996-v3.0-liquid.dtb \ + msm8996-v3.0-dtp.dtb \ + apq8096-v2-pmi8994-cdp.dtb \ + apq8096-v2-pmi8994-mtp.dtb \ + apq8096-v2-pmi8994-pmk8001-cdp.dtb \ + apq8096-v2-pmi8994-pm8004-cdp.dtb \ + apq8096-v2-pmi8994-pm8004-pmk8001-cdp.dtb \ + apq8096-v2-liquid.dtb \ + apq8096-v2-dragonboard.dtb \ + apq8096-v2-auto-dragonboard.dtb \ + apq8096-v3-pmi8994-cdp.dtb \ + apq8096-v3-pmi8994-mtp.dtb \ + apq8096-v3-pmi8994-pmk8001-cdp.dtb \ + apq8096-v3-pmi8994-pm8004-cdp.dtb \ + apq8096-v3-pmi8994-pm8004-pmk8001-cdp.dtb \ + apq8096-v3-pmi8996-cdp.dtb \ + apq8096-v3-pmi8996-mtp.dtb \ + apq8096-v3-liquid.dtb \ + apq8096-v3-dragonboard.dtb \ + apq8096-v3-sbc.dtb \ + apq8096-v3-auto-dragonboard.dtb \ + apq8096-v3-auto-adp.dtb \ + apq8096-v3-auto-cdp.dtb \ + apq8096-v3.0-pmi8994-cdp.dtb \ + apq8096-v3.0-pmi8994-mtp.dtb \ + apq8096-v3.0-pmi8994-pm8004-cdp.dtb \ + apq8096-v3.0-pmi8994-pm8004-pmk8001-cdp.dtb \ + apq8096-v3.0-pmi8994-pmk8001-cdp.dtb \ + apq8096-v3.0-pmi8996-cdp.dtb \ + apq8096-v3.0-pmi8996-mtp.dtb \ + apq8096-v3.0-liquid.dtb \ + apq8096-v3.0-dragonboard.dtb \ + apq8096-v3-pmi8994-mdm9x55-i2s-cdp.dtb \ + apq8096-v3-pmi8994-pm8004-mdm9x55-i2s-cdp.dtb \ + apq8096-v3-pmi8994-pm8004-pmk8001-mdm9x55-i2s-cdp.dtb \ + apq8096-v3-pmi8994-pmk8001-mdm9x55-i2s-cdp.dtb \ + apq8096-v3-pmi8996-mdm9x55-i2s-cdp.dtb \ + apq8096-v3-pmi8994-mdm9x55-i2s-mtp.dtb \ + apq8096-v3-pmi8994-mdm9x55-slimbus-mtp.dtb \ + apq8096-v3-pmi8996-mdm9x55-i2s-mtp.dtb \ + apq8096-v3-pmi8996-mdm9x55-slimbus-mtp.dtb \ + apq8096-v3-pmi8996-dragonboard.dtb \ + msm8996-auto-mizar.dtb -always := $(dtb-y) +dtb-$(CONFIG_MSM_GVM_QUIN) += vplatform-lfv-msm8996-telematics.dtb \ + vplatform-lfv-msm8996-ivi.dtb \ + vplatform-lfv-msm8996-baseline.dtb \ + vplatform-lfv-msm8996-ivi-la.dtb \ + vplatform-lfv-msm8996-ivi-lv-mt.dtb + +dtb-$(CONFIG_ARCH_MSM8998) += \ + apq8098-v2.1-mediabox.dtb \ + apq8098-freebox-proto.dtb \ + apq8098-freebox-oarfish.dtb \ + apq8098-freebox-batfish.dtb + +dtb-$(CONFIG_ARCH_MSMHAMSTER) += msmhamster-rumi.dtb + +dtb-$(CONFIG_ARCH_SDM660) += sdm660-sim.dtb \ + sdm660-internal-codec-cdp.dtb \ + sdm660-internal-codec-mtp.dtb \ + sdm660-internal-codec-rcm.dtb \ + sdm660-cdp.dtb \ + sdm660-mtp.dtb \ + sdm660-qrd.dtb \ + sdm660-rcm.dtb \ + sdm660-rumi.dtb \ + sdm660-pm660a-cdp.dtb \ + sdm660-pm660a-mtp.dtb \ + sdm660-pm660a-qrd.dtb \ + sdm660-pm660a-rcm.dtb \ + sdm660-pm660a-rumi.dtb \ + sdm660-internal-codec-pm660a-cdp.dtb \ + sdm660-internal-codec-pm660a-mtp.dtb \ + sdm660-internal-codec-pm660a-rcm.dtb \ + sdm660-pm660a-sim.dtb \ + sda660-cdp.dtb \ + sda660-mtp.dtb \ + sda660-rcm.dtb \ + sda660-pm660a-cdp.dtb \ + sda660-pm660a-mtp.dtb \ + sda660-pm660a-rcm.dtb \ + sda660-pm660a-qrd-hdk.dtb \ + sdm660-headset-jacktype-no-cdp.dtb \ + sdm660-headset-jacktype-no-rcm.dtb \ + sdm660-pm660a-headset-jacktype-no-cdp.dtb \ + sdm660-pm660a-headset-jacktype-no-rcm.dtb \ + sdm660-usbc-audio-mtp.dtb \ + sdm660-usbc-audio-rcm.dtb \ + sdm660-fhd-cdp.dtb \ + sdm660-pm660a-fhd-cdp.dtb \ + sdm658-mtp.dtb \ + sdm658-cdp.dtb \ + sdm658-rcm.dtb \ + sdm658-qrd.dtb \ + sdm658-pm660a-mtp.dtb \ + sdm658-pm660a-cdp.dtb \ + sdm658-pm660a-rcm.dtb \ + sdm658-pm660a-qrd.dtb \ + sdm658-internal-codec-mtp.dtb \ + sdm658-internal-codec-cdp.dtb \ + sdm658-internal-codec-rcm.dtb \ + sdm658-internal-codec-pm660a-mtp.dtb \ + sdm658-internal-codec-pm660a-cdp.dtb \ + sdm658-internal-codec-pm660a-rcm.dtb \ + sda658-cdp.dtb \ + sda658-mtp.dtb \ + sda658-rcm.dtb \ + sda658-pm660a-mtp.dtb \ + sda658-pm660a-cdp.dtb \ + sda658-pm660a-rcm.dtb \ + sdm636-cdp.dtb \ + sdm636-mtp.dtb \ + sdm636-qrd.dtb \ + sdm636-rcm.dtb \ + sdm636-headset-jacktype-no-cdp.dtb \ + sdm636-headset-jacktype-no-rcm.dtb \ + sdm636-internal-codec-cdp.dtb \ + sdm636-internal-codec-mtp.dtb \ + sdm636-internal-codec-pm660a-cdp.dtb \ + sdm636-internal-codec-pm660a-mtp.dtb \ + sdm636-internal-codec-pm660a-rcm.dtb \ + sdm636-internal-codec-rcm.dtb \ + sdm636-pm660a-headset-jacktype-no-cdp.dtb \ + sdm636-pm660a-headset-jacktype-no-rcm.dtb \ + sdm636-pm660a-cdp.dtb \ + sdm636-pm660a-mtp.dtb \ + sdm636-pm660a-qrd.dtb \ + sdm636-pm660a-rcm.dtb \ + sdm636-usbc-audio-mtp.dtb \ + sdm636-usbc-audio-rcm.dtb \ + sda636-cdp.dtb \ + sda636-mtp.dtb \ + sda636-rcm.dtb \ + sda636-pm660a-cdp.dtb \ + sda636-pm660a-mtp.dtb \ + sda636-pm660a-qrd-hdk.dtb \ + sda636-pm660a-rcm.dtb + +dtb-$(CONFIG_ARCH_SDM630) += sdm630-rumi.dtb \ + sdm630-pm660a-rumi.dtb \ + sdm630-mtp.dtb \ + sdm630-usbc-audio-mtp.dtb \ + sdm630-usbc-audio-rcm.dtb \ + sdm630-cdp.dtb \ + sdm630-rcm.dtb \ + sdm630-internal-codec-mtp.dtb \ + sdm630-internal-codec-cdp.dtb \ + sdm630-internal-codec-rcm.dtb \ + sdm630-pm660a-cdp.dtb \ + sdm630-pm660a-mtp.dtb \ + sdm630-pm660a-rcm.dtb \ + sdm630-pm660a-qrd.dtb \ + sdm630-internal-codec-pm660a-cdp.dtb \ + sdm630-internal-codec-pm660a-mtp.dtb \ + sdm630-internal-codec-pm660a-rcm.dtb \ + sda630-mtp.dtb \ + sda630-cdp.dtb \ + sda630-rcm.dtb \ + sda630-pm660a-mtp.dtb \ + sda630-pm660a-cdp.dtb \ + sda630-pm660a-rcm.dtb \ + sda630-pm660a-qrd-hdk.dtb \ + sdm630-headset-jacktype-no-cdp.dtb \ + sdm630-headset-jacktype-no-rcm.dtb \ + sdm630-pm660a-headset-jacktype-no-cdp.dtb \ + sdm630-pm660a-headset-jacktype-no-rcm.dtb + +ifeq ($(CONFIG_ARM64),y) +always := $(dtb-y) qcom_dtbs +always += $(dtbo-y) subdir-y := $(dts-dirs) -clean-files := *.dtb +else +targets += dtbs +targets += $(addprefix ../, $(dtb-y)) + +$(obj)/../%.dtb: $(src)/%.dts FORCE + $(call if_changed_dep,dtc) + +dtbs: $(addprefix $(obj)/../,$(dtb-y)) +endif +clean-files := *.dtbo *.dtb qcom_dtbs qcom_dtbs.cmpxz + +cmd_dtbs = ./scripts/dtbs.sh $@ $^ +quiet_cmd_dtbs = DTBS $@ + +DTB_ALIGN=32 +$(obj)/qcom_dtbs: $(addprefix $(obj)/,$(dtb-y)) + $(call cmd,dtbs) diff -ruw linux-4.4.115/arch/arm64/boot/Makefile linux-4.4.115-fbx/arch/arm64/boot/Makefile --- linux-4.4.115/arch/arm64/boot/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/boot/Makefile 2019-01-22 16:16:21.523228476 +0100 @@ -16,12 +16,23 @@ targets := Image Image.gz +DTB_NAMES := $(subst $\",,$(CONFIG_BUILD_ARM64_APPENDED_DTB_IMAGE_NAMES)) +ifneq ($(DTB_NAMES),) +DTB_LIST := $(addsuffix .dtb,$(DTB_NAMES)) +DTB_OBJS := $(addprefix $(obj)/dts/,$(DTB_LIST)) +else +DTB_OBJS := $(shell find -L $(obj)/dts/ -name \*.dtb) +endif + $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) $(obj)/Image.bz2: $(obj)/Image FORCE $(call if_changed,bzip2) +$(obj)/Image-dtb: $(obj)/Image $(DTB_OBJS) FORCE + $(call if_changed,cat) + $(obj)/Image.gz: $(obj)/Image FORCE $(call if_changed,gzip) @@ -34,6 +45,9 @@ $(obj)/Image.lzo: $(obj)/Image FORCE $(call if_changed,lzo) +$(obj)/Image.gz-dtb: $(obj)/Image.gz $(DTB_OBJS) FORCE + $(call if_changed,cat) + install: $(obj)/Image $(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \ $(obj)/Image System.map "$(INSTALL_PATH)" diff -ruw linux-4.4.115/arch/arm64/crypto/aes-glue.c linux-4.4.115-fbx/arch/arm64/crypto/aes-glue.c --- linux-4.4.115/arch/arm64/crypto/aes-glue.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/crypto/aes-glue.c 2019-01-22 16:16:21.535228585 +0100 @@ -294,7 +294,7 @@ .cra_blkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .ivsize = 0, .setkey = aes_setkey, .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, @@ -371,7 +371,7 @@ .cra_ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .ivsize = 0, .setkey = ablk_set_key, .encrypt = ablk_encrypt, .decrypt = ablk_decrypt, diff -ruw linux-4.4.115/arch/arm64/crypto/Kconfig linux-4.4.115-fbx/arch/arm64/crypto/Kconfig --- linux-4.4.115/arch/arm64/crypto/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/crypto/Kconfig 2019-01-22 16:16:21.531228549 +0100 @@ -23,6 +23,11 @@ depends on ARM64 && KERNEL_MODE_NEON select CRYPTO_HASH +config CRYPTO_POLY_HASH_ARM64_CE + tristate "poly_hash (for HEH encryption mode) using ARMv8 Crypto Extensions" + depends on ARM64 && KERNEL_MODE_NEON + select CRYPTO_HASH + config CRYPTO_AES_ARM64_CE tristate "AES core cipher using ARMv8 Crypto Extensions" depends on ARM64 && KERNEL_MODE_NEON diff -ruw linux-4.4.115/arch/arm64/crypto/Makefile linux-4.4.115-fbx/arch/arm64/crypto/Makefile --- linux-4.4.115/arch/arm64/crypto/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/crypto/Makefile 2019-01-22 16:16:21.531228549 +0100 @@ -17,6 +17,9 @@ obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o +obj-$(CONFIG_CRYPTO_POLY_HASH_ARM64_CE) += poly-hash-ce.o +poly-hash-ce-y := poly-hash-ce-glue.o poly-hash-ce-core.o + obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto diff -ruw linux-4.4.115/arch/arm64/crypto/sha1-ce-core.S linux-4.4.115-fbx/arch/arm64/crypto/sha1-ce-core.S --- linux-4.4.115/arch/arm64/crypto/sha1-ce-core.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/crypto/sha1-ce-core.S 2019-01-22 16:16:21.535228585 +0100 @@ -82,7 +82,8 @@ ldr dgb, [x0, #16] /* load sha1_ce_state::finalize */ - ldr w4, [x0, #:lo12:sha1_ce_offsetof_finalize] + ldr_l w4, sha1_ce_offsetof_finalize, x4 + ldr w4, [x0, x4] /* load input */ 0: ld1 {v8.4s-v11.4s}, [x1], #64 @@ -132,7 +133,8 @@ * the padding is handled by the C code in that case. */ cbz x4, 3f - ldr x4, [x0, #:lo12:sha1_ce_offsetof_count] + ldr_l w4, sha1_ce_offsetof_count, x4 + ldr x4, [x0, x4] movi v9.2d, #0 mov x8, #0x80000000 movi v10.2d, #0 diff -ruw linux-4.4.115/arch/arm64/crypto/sha1-ce-glue.c linux-4.4.115-fbx/arch/arm64/crypto/sha1-ce-glue.c --- linux-4.4.115/arch/arm64/crypto/sha1-ce-glue.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/crypto/sha1-ce-glue.c 2019-10-29 09:26:22.993196817 +0100 @@ -17,9 +17,6 @@ #include #include -#define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); - MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); @@ -32,6 +29,9 @@ asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, int blocks); +const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count); +const u32 sha1_ce_offsetof_finalize = offsetof(struct sha1_ce_state, finalize); + static int sha1_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -52,11 +52,6 @@ struct sha1_ce_state *sctx = shash_desc_ctx(desc); bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); - ASM_EXPORT(sha1_ce_offsetof_count, - offsetof(struct sha1_ce_state, sst.count)); - ASM_EXPORT(sha1_ce_offsetof_finalize, - offsetof(struct sha1_ce_state, finalize)); - /* * Allow the asm code to perform the finalization if there is no * partial data and the input is a round multiple of the block size. diff -ruw linux-4.4.115/arch/arm64/crypto/sha2-ce-core.S linux-4.4.115-fbx/arch/arm64/crypto/sha2-ce-core.S --- linux-4.4.115/arch/arm64/crypto/sha2-ce-core.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/crypto/sha2-ce-core.S 2019-01-22 16:16:21.535228585 +0100 @@ -88,7 +88,8 @@ ld1 {dgav.4s, dgbv.4s}, [x0] /* load sha256_ce_state::finalize */ - ldr w4, [x0, #:lo12:sha256_ce_offsetof_finalize] + ldr_l w4, sha256_ce_offsetof_finalize, x4 + ldr w4, [x0, x4] /* load input */ 0: ld1 {v16.4s-v19.4s}, [x1], #64 @@ -136,7 +137,8 @@ * the padding is handled by the C code in that case. */ cbz x4, 3f - ldr x4, [x0, #:lo12:sha256_ce_offsetof_count] + ldr_l w4, sha256_ce_offsetof_count, x4 + ldr x4, [x0, x4] movi v17.2d, #0 mov x8, #0x80000000 movi v18.2d, #0 diff -ruw linux-4.4.115/arch/arm64/crypto/sha2-ce-glue.c linux-4.4.115-fbx/arch/arm64/crypto/sha2-ce-glue.c --- linux-4.4.115/arch/arm64/crypto/sha2-ce-glue.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/crypto/sha2-ce-glue.c 2019-10-29 09:26:22.993196817 +0100 @@ -17,9 +17,6 @@ #include #include -#define ASM_EXPORT(sym, val) \ - asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); - MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); @@ -32,6 +29,11 @@ asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, int blocks); +const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, + sst.count); +const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state, + finalize); + static int sha256_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { @@ -52,11 +54,6 @@ struct sha256_ce_state *sctx = shash_desc_ctx(desc); bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); - ASM_EXPORT(sha256_ce_offsetof_count, - offsetof(struct sha256_ce_state, sst.count)); - ASM_EXPORT(sha256_ce_offsetof_finalize, - offsetof(struct sha256_ce_state, finalize)); - /* * Allow the asm code to perform the finalization if there is no * partial data and the input is a round multiple of the block size. diff -ruw linux-4.4.115/arch/arm64/include/asm/acpi.h linux-4.4.115-fbx/arch/arm64/include/asm/acpi.h --- linux-4.4.115/arch/arm64/include/asm/acpi.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/acpi.h 2019-01-22 16:16:21.535228585 +0100 @@ -87,9 +87,26 @@ static inline void acpi_init_cpus(void) { } #endif /* CONFIG_ACPI */ +#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL +bool acpi_parking_protocol_valid(int cpu); +void __init +acpi_set_mailbox_entry(int cpu, struct acpi_madt_generic_interrupt *processor); +#else +static inline bool acpi_parking_protocol_valid(int cpu) { return false; } +static inline void +acpi_set_mailbox_entry(int cpu, struct acpi_madt_generic_interrupt *processor) +{} +#endif + static inline const char *acpi_get_enable_method(int cpu) { - return acpi_psci_present() ? "psci" : NULL; + if (acpi_psci_present()) + return "psci"; + + if (acpi_parking_protocol_valid(cpu)) + return "parking-protocol"; + + return NULL; } #ifdef CONFIG_ACPI_APEI diff -ruw linux-4.4.115/arch/arm64/include/asm/alternative.h linux-4.4.115-fbx/arch/arm64/include/asm/alternative.h --- linux-4.4.115/arch/arm64/include/asm/alternative.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/alternative.h 2019-01-22 16:16:21.535228585 +0100 @@ -1,6 +1,9 @@ #ifndef __ASM_ALTERNATIVE_H #define __ASM_ALTERNATIVE_H +#include +#include + #ifndef __ASSEMBLY__ #include @@ -19,7 +22,6 @@ void __init apply_alternatives_all(void); void apply_alternatives(void *start, size_t length); -void free_alternatives_memory(void); #define ALTINSTR_ENTRY(feature) \ " .word 661b - .\n" /* label */ \ @@ -64,6 +66,8 @@ #else +#include + .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len .word \orig_offset - . .word \alt_offset - . @@ -87,55 +91,147 @@ .endm /* - * Begin an alternative code sequence. + * Alternative sequences + * + * The code for the case where the capability is not present will be + * assembled and linked as normal. There are no restrictions on this + * code. + * + * The code for the case where the capability is present will be + * assembled into a special section to be used for dynamic patching. + * Code for that case must: * - * The code that follows this macro will be assembled and linked as - * normal. There are no restrictions on this code. + * 1. Be exactly the same length (in bytes) as the default code + * sequence. + * + * 2. Not contain a branch target that is used outside of the + * alternative sequence it is defined in (branches into an + * alternative sequence are not fixed up). */ -.macro alternative_if_not cap, enable = 1 - .if \enable + +/* + * Begin an alternative code sequence. + */ +.macro alternative_if_not cap + .set .Lasm_alt_mode, 0 .pushsection .altinstructions, "a" altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f .popsection 661: - .endif +.endm + +.macro alternative_if cap + .set .Lasm_alt_mode, 1 + .pushsection .altinstructions, "a" + altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f + .popsection + .pushsection .altinstr_replacement, "ax" + .align 2 /* So GAS knows label 661 is suitably aligned */ +661: .endm /* - * Provide the alternative code sequence. - * - * The code that follows this macro is assembled into a special - * section to be used for dynamic patching. Code that follows this - * macro must: - * - * 1. Be exactly the same length (in bytes) as the default code - * sequence. - * - * 2. Not contain a branch target that is used outside of the - * alternative sequence it is defined in (branches into an - * alternative sequence are not fixed up). + * Provide the other half of the alternative code sequence. */ -.macro alternative_else, enable = 1 - .if \enable -662: .pushsection .altinstr_replacement, "ax" -663: +.macro alternative_else +662: + .if .Lasm_alt_mode==0 + .pushsection .altinstr_replacement, "ax" + .else + .popsection .endif +663: .endm /* * Complete an alternative code sequence. */ -.macro alternative_endif, enable = 1 - .if \enable -664: .popsection +.macro alternative_endif +664: + .if .Lasm_alt_mode==0 + .popsection + .endif .org . - (664b-663b) + (662b-661b) .org . - (662b-661b) + (664b-663b) - .endif +.endm + +/* + * Provides a trivial alternative or default sequence consisting solely + * of NOPs. The number of NOPs is chosen automatically to match the + * previous case. + */ +.macro alternative_else_nop_endif +alternative_else + nops (662b-661b) / AARCH64_INSN_SIZE +alternative_endif .endm #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) +.macro user_alt, label, oldinstr, newinstr, cond +9999: alternative_insn "\oldinstr", "\newinstr", \cond + _ASM_EXTABLE 9999b, \label +.endm + +/* + * Generate the assembly for UAO alternatives with exception table entries. + * This is complicated as there is no post-increment or pair versions of the + * unprivileged instructions, and USER() only works for single instructions. + */ +#ifdef CONFIG_ARM64_UAO + .macro uao_ldp l, reg1, reg2, addr, post_inc + alternative_if_not ARM64_HAS_UAO +8888: ldp \reg1, \reg2, [\addr], \post_inc; +8889: nop; + nop; + alternative_else + ldtr \reg1, [\addr]; + ldtr \reg2, [\addr, #8]; + add \addr, \addr, \post_inc; + alternative_endif + + _asm_extable 8888b,\l; + _asm_extable 8889b,\l; + .endm + + .macro uao_stp l, reg1, reg2, addr, post_inc + alternative_if_not ARM64_HAS_UAO +8888: stp \reg1, \reg2, [\addr], \post_inc; +8889: nop; + nop; + alternative_else + sttr \reg1, [\addr]; + sttr \reg2, [\addr, #8]; + add \addr, \addr, \post_inc; + alternative_endif + + _asm_extable 8888b,\l; + _asm_extable 8889b,\l; + .endm + + .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc + alternative_if_not ARM64_HAS_UAO +8888: \inst \reg, [\addr], \post_inc; + nop; + alternative_else + \alt_inst \reg, [\addr]; + add \addr, \addr, \post_inc; + alternative_endif + + _asm_extable 8888b,\l; + .endm +#else + .macro uao_ldp l, reg1, reg2, addr, post_inc + USER(\l, ldp \reg1, \reg2, [\addr], \post_inc) + .endm + .macro uao_stp l, reg1, reg2, addr, post_inc + USER(\l, stp \reg1, \reg2, [\addr], \post_inc) + .endm + .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc + USER(\l, \inst \reg, [\addr], \post_inc) + .endm +#endif #endif /* __ASSEMBLY__ */ diff -ruw linux-4.4.115/arch/arm64/include/asm/arch_gicv3.h linux-4.4.115-fbx/arch/arm64/include/asm/arch_gicv3.h --- linux-4.4.115/arch/arm64/include/asm/arch_gicv3.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/arch_gicv3.h 2019-01-22 16:16:21.535228585 +0100 @@ -103,7 +103,8 @@ u64 irqstat; asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat)); - dsb(sy); + /* As per the architecture specification */ + mb(); return irqstat; } @@ -132,6 +133,9 @@ static inline void gic_write_pmr(u32 val) { asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" ((u64)val)); + /* As per the architecture specification */ + isb(); + mb(); } static inline void gic_write_ctlr(u32 val) @@ -149,6 +153,9 @@ static inline void gic_write_sgi1r(u64 val) { asm volatile("msr_s " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val)); + /* As per the architecture specification */ + isb(); + mb(); } static inline u32 gic_read_sre(void) @@ -165,8 +172,8 @@ isb(); } -#define gic_read_typer(c) readq_relaxed(c) -#define gic_write_irouter(v, c) writeq_relaxed(v, c) +#define gic_read_typer(c) readq_relaxed_no_log(c) +#define gic_write_irouter(v, c) writeq_relaxed_no_log(v, c) #endif /* __ASSEMBLY__ */ #endif /* __ASM_ARCH_GICV3_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/arch_timer.h linux-4.4.115-fbx/arch/arm64/include/asm/arch_timer.h --- linux-4.4.115/arch/arm64/include/asm/arch_timer.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/arch_timer.h 2019-01-22 16:16:21.535228585 +0100 @@ -119,7 +119,14 @@ u64 cval; isb(); +#if IS_ENABLED(CONFIG_MSM_TIMER_LEAP) +#define L32_BITS 0x00000000FFFFFFFF + do { asm volatile("mrs %0, cntvct_el0" : "=r" (cval)); + } while ((cval & L32_BITS) == L32_BITS); +#else + asm volatile("mrs %0, cntvct_el0" : "=r" (cval)); +#endif return cval; } diff -ruw linux-4.4.115/arch/arm64/include/asm/assembler.h linux-4.4.115-fbx/arch/arm64/include/asm/assembler.h --- linux-4.4.115/arch/arm64/include/asm/assembler.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/assembler.h 2019-10-29 09:26:22.993196817 +0100 @@ -1,5 +1,5 @@ /* - * Based on arch/arm/include/asm/assembler.h + * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S * * Copyright (C) 1996-2000 Russell King * Copyright (C) 2012 ARM Ltd. @@ -23,6 +23,10 @@ #ifndef __ASM_ASSEMBLER_H #define __ASM_ASSEMBLER_H +#include +#include +#include +#include #include #include @@ -49,6 +53,27 @@ msr daifclr, #2 .endm + .macro save_and_disable_irq, flags + mrs \flags, daif + msr daifset, #2 + .endm + + .macro restore_irq, flags + msr daif, \flags + .endm + +/* + * Save/disable and restore interrupts. + */ + .macro save_and_disable_irqs, olddaif + mrs \olddaif, daif + disable_irq + .endm + + .macro restore_irqs, olddaif + msr daif, \olddaif + .endm + /* * Enable and disable debug exceptions. */ @@ -94,12 +119,28 @@ dmb \opt .endm +/* + * NOP sequence + */ + .macro nops, num + .rept \num + nop + .endr + .endm + +/* + * Emit an entry into the exception table + */ + .macro _asm_extable, from, to + .pushsection __ex_table, "a" + .align 3 + .long (\from - .), (\to - .) + .popsection + .endm + #define USER(l, x...) \ 9999: x; \ - .section __ex_table,"a"; \ - .align 3; \ - .quad 9999b,l; \ - .previous + _asm_extable 9999b, l /* * Register aliases. @@ -194,6 +235,133 @@ .endm /* + * @dst: Result of per_cpu(sym, smp_processor_id()) + * @sym: The name of the per-cpu variable + * @tmp: scratch register + */ + .macro adr_this_cpu, dst, sym, tmp + adr_l \dst, \sym + mrs \tmp, tpidr_el1 + add \dst, \dst, \tmp + .endm + + /* + * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) + * @sym: The name of the per-cpu variable + * @tmp: scratch register + */ + .macro ldr_this_cpu dst, sym, tmp + adr_l \dst, \sym + mrs \tmp, tpidr_el1 + ldr \dst, [\dst, \tmp] + .endm + +/* + * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) + */ + .macro vma_vm_mm, rd, rn + ldr \rd, [\rn, #VMA_VM_MM] + .endm + +/* + * mmid - get context id from mm pointer (mm->context.id) + */ + .macro mmid, rd, rn + ldr \rd, [\rn, #MM_CONTEXT_ID] + .endm + +/* + * dcache_line_size - get the minimum D-cache line size from the CTR register. + */ + .macro dcache_line_size, reg, tmp + mrs \tmp, ctr_el0 // read CTR + ubfm \tmp, \tmp, #16, #19 // cache line size encoding + mov \reg, #4 // bytes per word + lsl \reg, \reg, \tmp // actual cache line size + .endm + +/* + * icache_line_size - get the minimum I-cache line size from the CTR register. + */ + .macro icache_line_size, reg, tmp + mrs \tmp, ctr_el0 // read CTR + and \tmp, \tmp, #0xf // cache line size encoding + mov \reg, #4 // bytes per word + lsl \reg, \reg, \tmp // actual cache line size + .endm + +/* + * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map + */ + .macro tcr_set_idmap_t0sz, valreg, tmpreg +#ifndef CONFIG_ARM64_VA_BITS_48 + ldr_l \tmpreg, idmap_t0sz + bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH +#endif + .endm + +/* + * Macro to perform a data cache maintenance for the interval + * [kaddr, kaddr + size) + * + * op: operation passed to dc instruction + * domain: domain used in dsb instruciton + * kaddr: starting virtual address of the region + * size: size of the region + * Corrupts: kaddr, size, tmp1, tmp2 + */ + .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 + dcache_line_size \tmp1, \tmp2 + add \size, \kaddr, \size + sub \tmp2, \tmp1, #1 + bic \kaddr, \kaddr, \tmp2 +9998: + .if (\op == cvau || \op == cvac) +alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE + dc \op, \kaddr +alternative_else + dc civac, \kaddr +alternative_endif + .else + dc \op, \kaddr + .endif + add \kaddr, \kaddr, \tmp1 + cmp \kaddr, \size + b.lo 9998b + dsb \domain + .endm + +/* + * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present + */ + .macro reset_pmuserenr_el0, tmpreg + mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer + sbfx \tmpreg, \tmpreg, #8, #4 + cmp \tmpreg, #1 // Skip if no PMU present + b.lt 9000f + msr pmuserenr_el0, xzr // Disable PMU access from EL0 +9000: + .endm + +/* + * copy_page - copy src to dest using temp registers t1-t8 + */ + .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req +9998: ldp \t1, \t2, [\src] + ldp \t3, \t4, [\src, #16] + ldp \t5, \t6, [\src, #32] + ldp \t7, \t8, [\src, #48] + add \src, \src, #64 + stnp \t1, \t2, [\dest] + stnp \t3, \t4, [\dest, #16] + stnp \t5, \t6, [\dest, #32] + stnp \t7, \t8, [\dest, #48] + add \dest, \dest, #64 + tst \src, #(PAGE_SIZE - 1) + b.ne 9998b + .endm + +/* * Annotate a function as position independent, i.e., safe to be called before * the kernel virtual mapping is activated. */ @@ -204,4 +372,42 @@ .size __pi_##x, . - x; \ ENDPROC(x) + /* + * Emit a 64-bit absolute little endian symbol reference in a way that + * ensures that it will be resolved at build time, even when building a + * PIE binary. This requires cooperation from the linker script, which + * must emit the lo32/hi32 halves individually. + */ + .macro le64sym, sym + .long \sym\()_lo32 + .long \sym\()_hi32 + .endm + + /* + * mov_q - move an immediate constant into a 64-bit register using + * between 2 and 4 movz/movk instructions (depending on the + * magnitude and sign of the operand) + */ + .macro mov_q, reg, val + .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) + movz \reg, :abs_g1_s:\val + .else + .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) + movz \reg, :abs_g2_s:\val + .else + movz \reg, :abs_g3:\val + movk \reg, :abs_g2_nc:\val + .endif + movk \reg, :abs_g1_nc:\val + .endif + movk \reg, :abs_g0_nc:\val + .endm + +/* + * Return the current thread_info. + */ + .macro get_thread_info, rd + mrs \rd, sp_el0 + .endm + #endif /* __ASM_ASSEMBLER_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/barrier.h linux-4.4.115-fbx/arch/arm64/include/asm/barrier.h --- linux-4.4.115/arch/arm64/include/asm/barrier.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/barrier.h 2019-01-22 16:16:21.535228585 +0100 @@ -20,6 +20,9 @@ #ifndef __ASSEMBLY__ +#define __nops(n) ".rept " #n "\nnop\n.endr\n" +#define nops(n) asm volatile(__nops(n)) + #define sev() asm volatile("sev" : : : "memory") #define wfe() asm volatile("wfe" : : : "memory") #define wfi() asm volatile("wfi" : : : "memory") diff -ruw linux-4.4.115/arch/arm64/include/asm/boot.h linux-4.4.115-fbx/arch/arm64/include/asm/boot.h --- linux-4.4.115/arch/arm64/include/asm/boot.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/boot.h 2019-01-22 16:16:21.535228585 +0100 @@ -11,4 +11,10 @@ #define MIN_FDT_ALIGN 8 #define MAX_FDT_SIZE SZ_2M +/* + * arm64 requires the kernel image to placed + * TEXT_OFFSET bytes beyond a 2 MB aligned base + */ +#define MIN_KIMG_ALIGN SZ_2M + #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/bug.h linux-4.4.115-fbx/arch/arm64/include/asm/bug.h --- linux-4.4.115/arch/arm64/include/asm/bug.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/bug.h 2019-10-29 09:26:22.993196817 +0100 @@ -18,7 +18,7 @@ #ifndef _ARCH_ARM64_ASM_BUG_H #define _ARCH_ARM64_ASM_BUG_H -#include +#include #ifdef CONFIG_GENERIC_BUG #define HAVE_ARCH_BUG diff -ruw linux-4.4.115/arch/arm64/include/asm/cacheflush.h linux-4.4.115-fbx/arch/arm64/include/asm/cacheflush.h --- linux-4.4.115/arch/arm64/include/asm/cacheflush.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/cacheflush.h 2019-01-22 16:16:21.535228585 +0100 @@ -40,6 +40,10 @@ * the implementation assumes non-aliasing VIPT D-cache and (aliasing) * VIPT or ASID-tagged VIVT I-cache. * + * flush_cache_all() + * + * Unconditionally clean and invalidate the entire cache. + * * flush_cache_mm(mm) * * Clean and invalidate all user space cache entries @@ -65,9 +69,11 @@ * - kaddr - page address * - size - region size */ +extern void flush_cache_all(void); extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void flush_icache_range(unsigned long start, unsigned long end); extern void __flush_dcache_area(void *addr, size_t len); +extern void __clean_dcache_area_pou(void *addr, size_t len); extern long __flush_cache_user_range(unsigned long start, unsigned long end); static inline void flush_cache_mm(struct mm_struct *mm) @@ -85,6 +91,12 @@ extern void __dma_map_area(const void *, size_t, int); extern void __dma_unmap_area(const void *, size_t, int); extern void __dma_flush_range(const void *, const void *); +extern void __dma_inv_range(const void *, const void *); +extern void __dma_clean_range(const void *, const void *); + +#define dmac_flush_range __dma_flush_range +#define dmac_inv_range __dma_inv_range +#define dmac_clean_range __dma_clean_range /* * Copy user data from/to a page which is mapped into a different @@ -154,9 +166,21 @@ int set_memory_rw(unsigned long addr, int numpages); int set_memory_x(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages); +#ifdef CONFIG_KERNEL_TEXT_RDONLY +void set_kernel_text_ro(void); +#else +static inline void set_kernel_text_ro(void) { } +#endif #ifdef CONFIG_DEBUG_RODATA void mark_rodata_ro(void); #endif +#ifdef CONFIG_FREE_PAGES_RDONLY +#define mark_addr_rdonly(a) set_memory_ro((unsigned long)a, 1); +#define mark_addr_rdwrite(a) set_memory_rw((unsigned long)a, 1); +#else +#define mark_addr_rdonly(a) +#define mark_addr_rdwrite(a) +#endif #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/cache.h linux-4.4.115-fbx/arch/arm64/include/asm/cache.h --- linux-4.4.115/arch/arm64/include/asm/cache.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/cache.h 2019-01-22 16:16:21.535228585 +0100 @@ -18,17 +18,17 @@ #include -#define L1_CACHE_SHIFT 7 +#define L1_CACHE_SHIFT 6 #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) /* * Memory returned by kmalloc() may be used for DMA, so we must make - * sure that all such allocations are cache aligned. Otherwise, - * unrelated code may cause parts of the buffer to be read into the - * cache before the transfer is done, causing old data to be seen by - * the CPU. + * sure that all such allocations are aligned to the maximum *known* + * cache line size on ARMv8 systems. Otherwise, unrelated code may + * cause parts of the buffer to be read into the cache before the + * transfer is done, causing old data to be seen by the CPU. */ -#define ARCH_DMA_MINALIGN L1_CACHE_BYTES +#define ARCH_DMA_MINALIGN (128) #ifndef __ASSEMBLY__ diff -ruw linux-4.4.115/arch/arm64/include/asm/cmpxchg.h linux-4.4.115-fbx/arch/arm64/include/asm/cmpxchg.h --- linux-4.4.115/arch/arm64/include/asm/cmpxchg.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/cmpxchg.h 2019-01-22 16:16:21.535228585 +0100 @@ -19,7 +19,6 @@ #define __ASM_CMPXCHG_H #include -#include #include #include diff -ruw linux-4.4.115/arch/arm64/include/asm/cpufeature.h linux-4.4.115-fbx/arch/arm64/include/asm/cpufeature.h --- linux-4.4.115/arch/arm64/include/asm/cpufeature.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/cpufeature.h 2019-10-29 09:26:22.997196856 +0100 @@ -30,14 +30,22 @@ #define ARM64_HAS_LSE_ATOMICS 5 #define ARM64_WORKAROUND_CAVIUM_23154 6 #define ARM64_WORKAROUND_834220 7 -#define ARM64_WORKAROUND_CAVIUM_27456 8 - -#define ARM64_NCAPS 9 +#define ARM64_HAS_NO_HW_PREFETCH 8 +#define ARM64_HAS_UAO 9 +#define ARM64_ALT_PAN_NOT_UAO 10 + +#define ARM64_WORKAROUND_CAVIUM_27456 11 +#define ARM64_HAS_VIRT_HOST_EXTN 12 +#define ARM64_HARDEN_BRANCH_PREDICTOR 13 +#define ARM64_UNMAP_KERNEL_AT_EL0 14 +#define ARM64_NCAPS 15 #ifndef __ASSEMBLY__ #include +extern const char *machine_name; + /* CPU feature register tracking */ enum ftr_type { FTR_EXACT, /* Use a predefined safe value */ @@ -163,7 +171,9 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, const char *info); +void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps); void check_local_cpu_errata(void); +void __init enable_errata_workarounds(void); #ifdef CONFIG_HOTPLUG_CPU void verify_local_cpu_capabilities(void); @@ -177,7 +187,7 @@ static inline bool cpu_supports_mixed_endian_el0(void) { - return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); + return id_aa64mmfr0_mixed_endian_el0(read_cpuid(SYS_ID_AA64MMFR0_EL1)); } static inline bool system_supports_mixed_endian_el0(void) @@ -185,6 +195,12 @@ return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1)); } +static inline bool system_uses_ttbr0_pan(void) +{ + return IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN) && + !cpus_have_cap(ARM64_HAS_PAN); +} + #endif /* __ASSEMBLY__ */ #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/cpu.h linux-4.4.115-fbx/arch/arm64/include/asm/cpu.h --- linux-4.4.115/arch/arm64/include/asm/cpu.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/cpu.h 2019-01-22 16:16:21.535228585 +0100 @@ -36,6 +36,7 @@ u64 reg_id_aa64isar1; u64 reg_id_aa64mmfr0; u64 reg_id_aa64mmfr1; + u64 reg_id_aa64mmfr2; u64 reg_id_aa64pfr0; u64 reg_id_aa64pfr1; diff -ruw linux-4.4.115/arch/arm64/include/asm/cputype.h linux-4.4.115-fbx/arch/arm64/include/asm/cputype.h --- linux-4.4.115/arch/arm64/include/asm/cputype.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/cputype.h 2019-10-29 09:26:22.997196856 +0100 @@ -32,11 +32,9 @@ #define MPIDR_AFFINITY_LEVEL(mpidr, level) \ ((mpidr >> MPIDR_LEVEL_SHIFT(level)) & MPIDR_LEVEL_MASK) -#define read_cpuid(reg) ({ \ - u64 __val; \ - asm("mrs %0, " #reg : "=r" (__val)); \ - __val; \ -}) +#define MMFR0_16KGRAN_SIZE 15 +#define MMFR0_16KGRAN_SHFT 20 +#define MMFR0_EL1_16KGRAN_MASK (MMFR0_16KGRAN_SIZE << MMFR0_16KGRAN_SHFT) #define MIDR_REVISION_MASK 0xf #define MIDR_REVISION(midr) ((midr) & MIDR_REVISION_MASK) @@ -57,26 +55,65 @@ #define MIDR_IMPLEMENTOR(midr) \ (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) -#define MIDR_CPU_PART(imp, partnum) \ +#define MIDR_CPU_MODEL(imp, partnum) \ (((imp) << MIDR_IMPLEMENTOR_SHIFT) | \ (0xf << MIDR_ARCHITECTURE_SHIFT) | \ ((partnum) << MIDR_PARTNUM_SHIFT)) +#define MIDR_CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \ + MIDR_ARCHITECTURE_MASK) + +#define MIDR_IS_CPU_MODEL_RANGE(midr, model, rv_min, rv_max) \ +({ \ + u32 _model = (midr) & MIDR_CPU_MODEL_MASK; \ + u32 rv = (midr) & (MIDR_REVISION_MASK | MIDR_VARIANT_MASK); \ + \ + _model == (model) && rv >= (rv_min) && rv <= (rv_max); \ + }) + #define ARM_CPU_IMP_ARM 0x41 #define ARM_CPU_IMP_APM 0x50 #define ARM_CPU_IMP_CAVIUM 0x43 +#define ARM_CPU_IMP_QCOM 0x51 #define ARM_CPU_PART_AEM_V8 0xD0F #define ARM_CPU_PART_FOUNDATION 0xD00 #define ARM_CPU_PART_CORTEX_A57 0xD07 +#define ARM_CPU_PART_CORTEX_A72 0xD08 #define ARM_CPU_PART_CORTEX_A53 0xD03 +#define ARM_CPU_PART_CORTEX_A72 0xD08 +#define ARM_CPU_PART_CORTEX_A73 0xD09 +#define ARM_CPU_PART_CORTEX_A75 0xD0A +#define ARM_CPU_PART_KRYO2XX_GOLD 0x800 +#define ARM_CPU_PART_KRYO2XX_SILVER 0x801 +#define QCOM_CPU_PART_KRYO 0x200 #define APM_CPU_PART_POTENZA 0x000 #define CAVIUM_CPU_PART_THUNDERX 0x0A1 +#define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) +#define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) +#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) +#define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73) +#define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75) +#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) +#define MIDR_KRYO2XX_SILVER \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, ARM_CPU_PART_KRYO2XX_SILVER) +#define MIDR_KRYO2XX_GOLD \ + MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, ARM_CPU_PART_KRYO2XX_GOLD) +#define MIDR_QCOM_KRYO MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO) + #ifndef __ASSEMBLY__ +#include + +#define read_cpuid(reg) ({ \ + u64 __val; \ + asm("mrs_s %0, " __stringify(reg) : "=r" (__val)); \ + __val; \ +}) + /* * The CPU ID never changes at run time, so we might as well tell the * compiler that it's constant. Use this function to read the CPU ID @@ -84,12 +121,12 @@ */ static inline u32 __attribute_const__ read_cpuid_id(void) { - return read_cpuid(MIDR_EL1); + return read_cpuid(SYS_MIDR_EL1); } static inline u64 __attribute_const__ read_cpuid_mpidr(void) { - return read_cpuid(MPIDR_EL1); + return read_cpuid(SYS_MPIDR_EL1); } static inline unsigned int __attribute_const__ read_cpuid_implementor(void) @@ -104,7 +141,7 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void) { - return read_cpuid(CTR_EL0); + return read_cpuid(SYS_CTR_EL0); } #endif /* __ASSEMBLY__ */ diff -ruw linux-4.4.115/arch/arm64/include/asm/debug-monitors.h linux-4.4.115-fbx/arch/arm64/include/asm/debug-monitors.h --- linux-4.4.115/arch/arm64/include/asm/debug-monitors.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/debug-monitors.h 2019-01-22 16:16:21.539228621 +0100 @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -47,19 +48,6 @@ #define BREAK_INSTR_SIZE AARCH64_INSN_SIZE /* - * #imm16 values used for BRK instruction generation - * Allowed values for kgbd are 0x400 - 0x7ff - * 0x100: for triggering a fault on purpose (reserved) - * 0x400: for dynamic BRK instruction - * 0x401: for compile time BRK instruction - * 0x800: kernel-mode BUG() and WARN() traps - */ -#define FAULT_BRK_IMM 0x100 -#define KGDB_DYN_DBG_BRK_IMM 0x400 -#define KGDB_COMPILED_DBG_BRK_IMM 0x401 -#define BUG_BRK_IMM 0x800 - -/* * BRK instruction encoding * The #imm16 value should be placed at bits[20:5] within BRK ins */ @@ -78,6 +66,11 @@ #define CACHE_FLUSH_IS_SAFE 1 +/* kprobes BRK opcodes with ESR encoding */ +#define BRK64_ESR_MASK 0xFFFF +#define BRK64_ESR_KPROBES 0x0004 +#define BRK64_OPCODE_KPROBES (AARCH64_BREAK_MON | (BRK64_ESR_KPROBES << 5)) + /* AArch32 */ #define DBG_ESR_EVT_BKPT 0x4 #define DBG_ESR_EVT_VECC 0x5 diff -ruw linux-4.4.115/arch/arm64/include/asm/device.h linux-4.4.115-fbx/arch/arm64/include/asm/device.h --- linux-4.4.115/arch/arm64/include/asm/device.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/device.h 2019-01-22 16:16:21.539228621 +0100 @@ -17,14 +17,24 @@ #define __ASM_DEVICE_H struct dev_archdata { - struct dma_map_ops *dma_ops; + const struct dma_map_ops *dma_ops; #ifdef CONFIG_IOMMU_API void *iommu; /* private IOMMU data */ #endif bool dma_coherent; +#ifdef CONFIG_ARM64_DMA_USE_IOMMU + struct dma_iommu_mapping *mapping; +#endif }; struct pdev_archdata { + u64 dma_mask; }; +#ifdef CONFIG_ARM64_DMA_USE_IOMMU +#define to_dma_iommu_mapping(dev) ((dev)->archdata.mapping) +#else +#define to_dma_iommu_mapping(dev) NULL +#endif + #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/dma-mapping.h linux-4.4.115-fbx/arch/arm64/include/asm/dma-mapping.h --- linux-4.4.115/arch/arm64/include/asm/dma-mapping.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/dma-mapping.h 2019-01-22 16:16:21.539228621 +0100 @@ -27,7 +27,7 @@ #define DMA_ERROR_CODE (~(dma_addr_t)0) extern struct dma_map_ops dummy_dma_ops; -static inline struct dma_map_ops *__generic_dma_ops(struct device *dev) +static inline const struct dma_map_ops *__generic_dma_ops(struct device *dev) { if (dev && dev->archdata.dma_ops) return dev->archdata.dma_ops; @@ -39,7 +39,7 @@ return &dummy_dma_ops; } -static inline struct dma_map_ops *get_dma_ops(struct device *dev) +static inline const struct dma_map_ops *get_dma_ops(struct device *dev) { if (xen_initial_domain()) return xen_dma_ops; @@ -47,6 +47,12 @@ return __generic_dma_ops(dev); } +static inline void set_dma_ops(struct device *dev, + const struct dma_map_ops *dma_ops) +{ + dev->archdata.dma_ops = dma_ops; +} + void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, struct iommu_ops *iommu, bool coherent); #define arch_setup_dma_ops arch_setup_dma_ops diff -ruw linux-4.4.115/arch/arm64/include/asm/efi.h linux-4.4.115-fbx/arch/arm64/include/asm/efi.h --- linux-4.4.115/arch/arm64/include/asm/efi.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/efi.h 2019-01-22 16:16:21.539228621 +0100 @@ -1,8 +1,11 @@ #ifndef _ASM_EFI_H #define _ASM_EFI_H +#include #include +#include #include +#include #ifdef CONFIG_EFI extern void efi_init(void); @@ -10,6 +13,8 @@ #define efi_init() #endif +int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); + #define efi_call_virt(f, ...) \ ({ \ efi_##f##_t *__f; \ @@ -63,6 +68,34 @@ * Services are enabled and the EFI_RUNTIME_SERVICES bit set. */ +static inline void efi_set_pgd(struct mm_struct *mm) +{ + __switch_mm(mm); + + if (system_uses_ttbr0_pan()) { + if (mm != current->active_mm) { + /* + * Update the current thread's saved ttbr0 since it is + * restored as part of a return from exception. Enable + * access to the valid TTBR0_EL1 and invoke the errata + * workaround directly since there is no return from + * exception when invoking the EFI run-time services. + */ + update_saved_ttbr0(current, mm); + uaccess_ttbr0_enable(); + post_ttbr_update_workaround(); + } else { + /* + * Defer the switch to the current thread's TTBR0_EL1 + * until uaccess_enable(). Restore the current + * thread's saved ttbr0 corresponding to its active_mm + */ + uaccess_ttbr0_disable(); + update_saved_ttbr0(current, current->active_mm); + } + } +} + void efi_virtmap_load(void); void efi_virtmap_unload(void); diff -ruw linux-4.4.115/arch/arm64/include/asm/elf.h linux-4.4.115-fbx/arch/arm64/include/asm/elf.h --- linux-4.4.115/arch/arm64/include/asm/elf.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/elf.h 2019-01-22 16:16:21.539228621 +0100 @@ -23,15 +23,7 @@ */ #include #include - -typedef unsigned long elf_greg_t; - -#define ELF_NGREG (sizeof(struct user_pt_regs) / sizeof(elf_greg_t)) -#define ELF_CORE_COPY_REGS(dest, regs) \ - *(struct user_pt_regs *)&(dest) = (regs)->user_regs; - -typedef elf_greg_t elf_gregset_t[ELF_NGREG]; -typedef struct user_fpsimd_state elf_fpregset_t; +#include /* * AArch64 static relocation types. @@ -86,6 +78,8 @@ #define R_AARCH64_MOVW_PREL_G2_NC 292 #define R_AARCH64_MOVW_PREL_G3 293 +#define R_AARCH64_RELATIVE 1027 + /* * These are used to set parameters in the core dumps. */ @@ -126,6 +120,17 @@ */ #define ELF_ET_DYN_BASE (2 * TASK_SIZE_64 / 3) +#ifndef __ASSEMBLY__ + +typedef unsigned long elf_greg_t; + +#define ELF_NGREG (sizeof(struct user_pt_regs) / sizeof(elf_greg_t)) +#define ELF_CORE_COPY_REGS(dest, regs) \ + *(struct user_pt_regs *)&(dest) = (regs)->user_regs; + +typedef elf_greg_t elf_gregset_t[ELF_NGREG]; +typedef struct user_fpsimd_state elf_fpregset_t; + /* * When the program starts, a1 contains a pointer to a function to be * registered with atexit, as per the SVR4 ABI. A value of 0 means we have no @@ -165,7 +170,7 @@ #ifdef CONFIG_COMPAT /* PIE load location for compat arm. Must match ARM ELF_ET_DYN_BASE. */ -#define COMPAT_ELF_ET_DYN_BASE 0x000400000UL +#define COMPAT_ELF_ET_DYN_BASE (2 * TASK_SIZE_32 / 3) /* AArch32 registers. */ #define COMPAT_ELF_NGREG 18 @@ -178,7 +183,11 @@ ((x)->e_flags & EF_ARM_EABI_MASK)) #define compat_start_thread compat_start_thread -#define COMPAT_SET_PERSONALITY(ex) set_thread_flag(TIF_32BIT); +#define COMPAT_SET_PERSONALITY(ex) \ +do { \ + set_thread_flag(TIF_32BIT); \ +} while (0) + #define COMPAT_ARCH_DLINFO extern int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp); @@ -187,4 +196,6 @@ #endif /* CONFIG_COMPAT */ +#endif /* !__ASSEMBLY__ */ + #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/esr.h linux-4.4.115-fbx/arch/arm64/include/asm/esr.h --- linux-4.4.115/arch/arm64/include/asm/esr.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/esr.h 2019-01-22 16:16:21.539228621 +0100 @@ -74,6 +74,7 @@ #define ESR_ELx_EC_SHIFT (26) #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT) +#define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) #define ESR_ELx_IL (UL(1) << 25) #define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) @@ -108,6 +109,46 @@ ((ESR_ELx_EC_BRK64 << ESR_ELx_EC_SHIFT) | ESR_ELx_IL | \ ((imm) & 0xffff)) +/* ISS field definitions for System instruction traps */ +#define ESR_ELx_SYS64_ISS_RES0_SHIFT 22 +#define ESR_ELx_SYS64_ISS_RES0_MASK (UL(0x7) << ESR_ELx_SYS64_ISS_RES0_SHIFT) +#define ESR_ELx_SYS64_ISS_DIR_MASK 0x1 +#define ESR_ELx_SYS64_ISS_DIR_READ 0x1 +#define ESR_ELx_SYS64_ISS_DIR_WRITE 0x0 + +#define ESR_ELx_SYS64_ISS_RT_SHIFT 5 +#define ESR_ELx_SYS64_ISS_RT_MASK (UL(0x1f) << ESR_ELx_SYS64_ISS_RT_SHIFT) +#define ESR_ELx_SYS64_ISS_CRM_SHIFT 1 +#define ESR_ELx_SYS64_ISS_CRM_MASK (UL(0xf) << ESR_ELx_SYS64_ISS_CRM_SHIFT) +#define ESR_ELx_SYS64_ISS_CRN_SHIFT 10 +#define ESR_ELx_SYS64_ISS_CRN_MASK (UL(0xf) << ESR_ELx_SYS64_ISS_CRN_SHIFT) +#define ESR_ELx_SYS64_ISS_OP1_SHIFT 14 +#define ESR_ELx_SYS64_ISS_OP1_MASK (UL(0x7) << ESR_ELx_SYS64_ISS_OP1_SHIFT) +#define ESR_ELx_SYS64_ISS_OP2_SHIFT 17 +#define ESR_ELx_SYS64_ISS_OP2_MASK (UL(0x7) << ESR_ELx_SYS64_ISS_OP2_SHIFT) +#define ESR_ELx_SYS64_ISS_OP0_SHIFT 20 +#define ESR_ELx_SYS64_ISS_OP0_MASK (UL(0x3) << ESR_ELx_SYS64_ISS_OP0_SHIFT) +#define ESR_ELx_SYS64_ISS_SYS_MASK (ESR_ELx_SYS64_ISS_OP0_MASK | \ + ESR_ELx_SYS64_ISS_OP1_MASK | \ + ESR_ELx_SYS64_ISS_OP2_MASK | \ + ESR_ELx_SYS64_ISS_CRN_MASK | \ + ESR_ELx_SYS64_ISS_CRM_MASK) +#define ESR_ELx_SYS64_ISS_SYS_VAL(op0, op1, op2, crn, crm) \ + (((op0) << ESR_ELx_SYS64_ISS_OP0_SHIFT) | \ + ((op1) << ESR_ELx_SYS64_ISS_OP1_SHIFT) | \ + ((op2) << ESR_ELx_SYS64_ISS_OP2_SHIFT) | \ + ((crn) << ESR_ELx_SYS64_ISS_CRN_SHIFT) | \ + ((crm) << ESR_ELx_SYS64_ISS_CRM_SHIFT)) + +#define ESR_ELx_SYS64_ISS_SYS_OP_MASK (ESR_ELx_SYS64_ISS_SYS_MASK | \ + ESR_ELx_SYS64_ISS_DIR_MASK) + +#define ESR_ELx_SYS64_ISS_SYS_CNTVCT (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 2, 14, 0) | \ + ESR_ELx_SYS64_ISS_DIR_READ) + +#define ESR_ELx_SYS64_ISS_SYS_CNTFRQ (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 0, 14, 0) | \ + ESR_ELx_SYS64_ISS_DIR_READ) + #ifndef __ASSEMBLY__ #include diff -ruw linux-4.4.115/arch/arm64/include/asm/exception.h linux-4.4.115-fbx/arch/arm64/include/asm/exception.h --- linux-4.4.115/arch/arm64/include/asm/exception.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/exception.h 2019-01-22 16:16:21.539228621 +0100 @@ -18,7 +18,7 @@ #ifndef __ASM_EXCEPTION_H #define __ASM_EXCEPTION_H -#include +#include #define __exception __attribute__((section(".exception.text"))) #ifdef CONFIG_FUNCTION_GRAPH_TRACER diff -ruw linux-4.4.115/arch/arm64/include/asm/exec.h linux-4.4.115-fbx/arch/arm64/include/asm/exec.h --- linux-4.4.115/arch/arm64/include/asm/exec.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/exec.h 2019-01-22 16:16:21.539228621 +0100 @@ -18,6 +18,9 @@ #ifndef __ASM_EXEC_H #define __ASM_EXEC_H +#include + extern unsigned long arch_align_stack(unsigned long sp); +void uao_thread_switch(struct task_struct *next); #endif /* __ASM_EXEC_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/fixmap.h linux-4.4.115-fbx/arch/arm64/include/asm/fixmap.h --- linux-4.4.115/arch/arm64/include/asm/fixmap.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/fixmap.h 2019-01-22 16:16:21.539228621 +0100 @@ -50,6 +50,11 @@ FIX_EARLYCON_MEM_BASE, FIX_TEXT_POKE0, +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 + FIX_ENTRY_TRAMP_DATA, + FIX_ENTRY_TRAMP_TEXT, +#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT)) +#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ __end_of_permanent_fixed_addresses, /* @@ -62,6 +67,16 @@ FIX_BTMAP_END = __end_of_permanent_fixed_addresses, FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, + + /* + * Used for kernel page table creation, so unmapped memory may be used + * for tables. + */ + FIX_PTE, + FIX_PMD, + FIX_PUD, + FIX_PGD, + __end_of_fixed_addresses }; diff -ruw linux-4.4.115/arch/arm64/include/asm/fpsimd.h linux-4.4.115-fbx/arch/arm64/include/asm/fpsimd.h --- linux-4.4.115/arch/arm64/include/asm/fpsimd.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/fpsimd.h 2019-01-22 16:16:21.539228621 +0100 @@ -81,6 +81,14 @@ u32 num_regs); extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state); +#ifdef CONFIG_ENABLE_FP_SIMD_SETTINGS +extern void fpsimd_disable_trap(void); +extern void fpsimd_enable_trap(void); +#else +static inline void fpsimd_disable_trap(void) {} +static inline void fpsimd_enable_trap(void) {} +#endif + #endif #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/ftrace.h linux-4.4.115-fbx/arch/arm64/include/asm/ftrace.h --- linux-4.4.115/arch/arm64/include/asm/ftrace.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/ftrace.h 2019-01-22 16:16:21.539228621 +0100 @@ -28,6 +28,8 @@ extern unsigned long ftrace_graph_call; +extern void return_to_handler(void); + static inline unsigned long ftrace_call_adjust(unsigned long addr) { /* diff -ruw linux-4.4.115/arch/arm64/include/asm/futex.h linux-4.4.115-fbx/arch/arm64/include/asm/futex.h --- linux-4.4.115/arch/arm64/include/asm/futex.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/futex.h 2019-10-29 09:26:22.997196856 +0100 @@ -21,15 +21,12 @@ #include #include -#include -#include #include -#include #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \ +do { \ + uaccess_enable(); \ asm volatile( \ - ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ " prfm pstl1strm, %2\n" \ "1: ldxr %w1, %2\n" \ insn "\n" \ @@ -42,15 +39,13 @@ "4: mov %w0, %w5\n" \ " b 3b\n" \ " .popsection\n" \ -" .pushsection __ex_table,\"a\"\n" \ -" .align 3\n" \ -" .quad 1b, 4b, 2b, 4b\n" \ -" .popsection\n" \ - ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \ : "r" (oparg), "Ir" (-EFAULT) \ - : "memory") + : "memory"); \ + uaccess_disable(); \ +} while (0) static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) @@ -120,8 +115,8 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; + uaccess_enable(); asm volatile("// futex_atomic_cmpxchg_inatomic\n" -ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN) " prfm pstl1strm, %2\n" "1: ldxr %w1, %2\n" " sub %w3, %w1, %w4\n" @@ -134,14 +129,12 @@ "4: mov %w0, %w6\n" " b 3b\n" " .popsection\n" -" .pushsection __ex_table,\"a\"\n" -" .align 3\n" -" .quad 1b, 4b, 2b, 4b\n" -" .popsection\n" -ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN) + _ASM_EXTABLE(1b, 4b) + _ASM_EXTABLE(2b, 4b) : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp) : "r" (oldval), "r" (newval), "Ir" (-EFAULT) : "memory"); + uaccess_disable(); *uval = val; return ret; diff -ruw linux-4.4.115/arch/arm64/include/asm/hardirq.h linux-4.4.115-fbx/arch/arm64/include/asm/hardirq.h --- linux-4.4.115/arch/arm64/include/asm/hardirq.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/hardirq.h 2019-01-22 16:16:21.539228621 +0100 @@ -20,7 +20,7 @@ #include #include -#define NR_IPI 5 +#define NR_IPI 7 typedef struct { unsigned int __softirq_pending; diff -ruw linux-4.4.115/arch/arm64/include/asm/hw_breakpoint.h linux-4.4.115-fbx/arch/arm64/include/asm/hw_breakpoint.h --- linux-4.4.115/arch/arm64/include/asm/hw_breakpoint.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/hw_breakpoint.h 2019-01-22 16:16:21.539228621 +0100 @@ -68,7 +68,11 @@ /* Lengths */ #define ARM_BREAKPOINT_LEN_1 0x1 #define ARM_BREAKPOINT_LEN_2 0x3 +#define ARM_BREAKPOINT_LEN_3 0x7 #define ARM_BREAKPOINT_LEN_4 0xf +#define ARM_BREAKPOINT_LEN_5 0x1f +#define ARM_BREAKPOINT_LEN_6 0x3f +#define ARM_BREAKPOINT_LEN_7 0x7f #define ARM_BREAKPOINT_LEN_8 0xff /* Kernel stepping */ @@ -110,7 +114,7 @@ struct pmu; extern int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, - int *gen_len, int *gen_type); + int *gen_len, int *gen_type, int *offset); extern int arch_check_bp_in_kernelspace(struct perf_event *bp); extern int arch_validate_hwbkpt_settings(struct perf_event *bp); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, diff -ruw linux-4.4.115/arch/arm64/include/asm/insn.h linux-4.4.115-fbx/arch/arm64/include/asm/insn.h --- linux-4.4.115/arch/arm64/include/asm/insn.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/insn.h 2019-01-22 16:16:21.539228621 +0100 @@ -120,6 +120,29 @@ AARCH64_INSN_REG_SP = 31 /* Stack pointer: as load/store base reg */ }; +enum aarch64_insn_special_register { + AARCH64_INSN_SPCLREG_SPSR_EL1 = 0xC200, + AARCH64_INSN_SPCLREG_ELR_EL1 = 0xC201, + AARCH64_INSN_SPCLREG_SP_EL0 = 0xC208, + AARCH64_INSN_SPCLREG_SPSEL = 0xC210, + AARCH64_INSN_SPCLREG_CURRENTEL = 0xC212, + AARCH64_INSN_SPCLREG_DAIF = 0xDA11, + AARCH64_INSN_SPCLREG_NZCV = 0xDA10, + AARCH64_INSN_SPCLREG_FPCR = 0xDA20, + AARCH64_INSN_SPCLREG_DSPSR_EL0 = 0xDA28, + AARCH64_INSN_SPCLREG_DLR_EL0 = 0xDA29, + AARCH64_INSN_SPCLREG_SPSR_EL2 = 0xE200, + AARCH64_INSN_SPCLREG_ELR_EL2 = 0xE201, + AARCH64_INSN_SPCLREG_SP_EL1 = 0xE208, + AARCH64_INSN_SPCLREG_SPSR_INQ = 0xE218, + AARCH64_INSN_SPCLREG_SPSR_ABT = 0xE219, + AARCH64_INSN_SPCLREG_SPSR_UND = 0xE21A, + AARCH64_INSN_SPCLREG_SPSR_FIQ = 0xE21B, + AARCH64_INSN_SPCLREG_SPSR_EL3 = 0xF200, + AARCH64_INSN_SPCLREG_ELR_EL3 = 0xF201, + AARCH64_INSN_SPCLREG_SP_EL2 = 0xF210 +}; + enum aarch64_insn_variant { AARCH64_INSN_VARIANT_32BIT, AARCH64_INSN_VARIANT_64BIT @@ -223,8 +246,15 @@ static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ { return (val); } +__AARCH64_INSN_FUNCS(adr_adrp, 0x1F000000, 0x10000000) +__AARCH64_INSN_FUNCS(prfm_lit, 0xFF000000, 0xD8000000) __AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800) __AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800) +__AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000) +__AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000) +__AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000) +__AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000) +__AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000) __AARCH64_INSN_FUNCS(stp_post, 0x7FC00000, 0x28800000) __AARCH64_INSN_FUNCS(ldp_post, 0x7FC00000, 0x28C00000) __AARCH64_INSN_FUNCS(stp_pre, 0x7FC00000, 0x29800000) @@ -273,10 +303,15 @@ __AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) __AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) __AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) +__AARCH64_INSN_FUNCS(exception, 0xFF000000, 0xD4000000) __AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F) __AARCH64_INSN_FUNCS(br, 0xFFFFFC1F, 0xD61F0000) __AARCH64_INSN_FUNCS(blr, 0xFFFFFC1F, 0xD63F0000) __AARCH64_INSN_FUNCS(ret, 0xFFFFFC1F, 0xD65F0000) +__AARCH64_INSN_FUNCS(eret, 0xFFFFFFFF, 0xD69F03E0) +__AARCH64_INSN_FUNCS(mrs, 0xFFF00000, 0xD5300000) +__AARCH64_INSN_FUNCS(msr_imm, 0xFFF8F01F, 0xD500401F) +__AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000) #undef __AARCH64_INSN_FUNCS @@ -286,6 +321,8 @@ int aarch64_insn_read(void *addr, u32 *insnp); int aarch64_insn_write(void *addr, u32 insn); enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); +bool aarch64_insn_uses_literal(u32 insn); +bool aarch64_insn_is_branch(u32 insn); u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn); u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, u32 insn, u64 imm); @@ -367,9 +404,13 @@ #define A32_RT_OFFSET 12 #define A32_RT2_OFFSET 0 +u32 aarch64_insn_extract_system_reg(u32 insn); u32 aarch32_insn_extract_reg_num(u32 insn, int offset); u32 aarch32_insn_mcr_extract_opc2(u32 insn); u32 aarch32_insn_mcr_extract_crm(u32 insn); + +typedef bool (pstate_check_t)(unsigned long); +extern pstate_check_t * const aarch32_opcode_cond_checks[16]; #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/io.h linux-4.4.115-fbx/arch/arm64/include/asm/io.h --- linux-4.4.115/arch/arm64/include/asm/io.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/io.h 2019-01-22 16:16:21.539228621 +0100 @@ -31,38 +31,35 @@ #include #include #include +#include #include /* * Generic IO read/write. These perform native-endian accesses. + * that some architectures will want to re-define __raw_{read,write}w. */ -#define __raw_writeb __raw_writeb -static inline void __raw_writeb(u8 val, volatile void __iomem *addr) +static inline void __raw_writeb_no_log(u8 val, volatile void __iomem *addr) { asm volatile("strb %w0, [%1]" : : "r" (val), "r" (addr)); } -#define __raw_writew __raw_writew -static inline void __raw_writew(u16 val, volatile void __iomem *addr) +static inline void __raw_writew_no_log(u16 val, volatile void __iomem *addr) { asm volatile("strh %w0, [%1]" : : "r" (val), "r" (addr)); } -#define __raw_writel __raw_writel -static inline void __raw_writel(u32 val, volatile void __iomem *addr) +static inline void __raw_writel_no_log(u32 val, volatile void __iomem *addr) { asm volatile("str %w0, [%1]" : : "r" (val), "r" (addr)); } -#define __raw_writeq __raw_writeq -static inline void __raw_writeq(u64 val, volatile void __iomem *addr) +static inline void __raw_writeq_no_log(u64 val, volatile void __iomem *addr) { asm volatile("str %0, [%1]" : : "r" (val), "r" (addr)); } -#define __raw_readb __raw_readb -static inline u8 __raw_readb(const volatile void __iomem *addr) +static inline u8 __raw_readb_no_log(const volatile void __iomem *addr) { u8 val; asm volatile(ALTERNATIVE("ldrb %w0, [%1]", @@ -72,8 +69,7 @@ return val; } -#define __raw_readw __raw_readw -static inline u16 __raw_readw(const volatile void __iomem *addr) +static inline u16 __raw_readw_no_log(const volatile void __iomem *addr) { u16 val; @@ -84,8 +80,7 @@ return val; } -#define __raw_readl __raw_readl -static inline u32 __raw_readl(const volatile void __iomem *addr) +static inline u32 __raw_readl_no_log(const volatile void __iomem *addr) { u32 val; asm volatile(ALTERNATIVE("ldr %w0, [%1]", @@ -95,8 +90,7 @@ return val; } -#define __raw_readq __raw_readq -static inline u64 __raw_readq(const volatile void __iomem *addr) +static inline u64 __raw_readq_no_log(const volatile void __iomem *addr) { u64 val; asm volatile(ALTERNATIVE("ldr %0, [%1]", @@ -106,6 +100,46 @@ return val; } +/* + * There may be cases when clients don't want to support or can't support the + * logging, The appropriate functions can be used but clinets should carefully + * consider why they can't support the logging + */ + +#define __raw_write_logged(v, a, _t) ({ \ + int _ret; \ + volatile void __iomem *_a = (a); \ + void *_addr = (void __force *)(_a); \ + _ret = uncached_logk(LOGK_WRITEL, _addr); \ + ETB_WAYPOINT; \ + __raw_write##_t##_no_log((v), _a); \ + if (_ret) \ + LOG_BARRIER; \ + }) + +#define __raw_writeb(v, a) __raw_write_logged((v), a, b) +#define __raw_writew(v, a) __raw_write_logged((v), a, w) +#define __raw_writel(v, a) __raw_write_logged((v), a, l) +#define __raw_writeq(v, a) __raw_write_logged((v), a, q) + +#define __raw_read_logged(a, _l, _t) ({ \ + _t __a; \ + const volatile void __iomem *_a = (const volatile void __iomem *)(a); \ + void *_addr = (void __force *)(_a); \ + int _ret; \ + _ret = uncached_logk(LOGK_READL, _addr); \ + ETB_WAYPOINT; \ + __a = __raw_read##_l##_no_log(_a); \ + if (_ret) \ + LOG_BARRIER; \ + __a; \ + }) + +#define __raw_readb(a) __raw_read_logged((a), b, u8) +#define __raw_readw(a) __raw_read_logged((a), w, u16) +#define __raw_readl(a) __raw_read_logged((a), l, u32) +#define __raw_readq(a) __raw_read_logged((a), q, u64) + /* IO barriers */ #define __iormb() rmb() #define __iowmb() wmb() @@ -127,6 +161,16 @@ #define writel_relaxed(v,c) ((void)__raw_writel((__force u32)cpu_to_le32(v),(c))) #define writeq_relaxed(v,c) ((void)__raw_writeq((__force u64)cpu_to_le64(v),(c))) +#define readb_relaxed_no_log(c) ({ u8 __v = __raw_readb_no_log(c); __v; }) +#define readw_relaxed_no_log(c) ({ u16 __v = le16_to_cpu((__force __le16)__raw_readw_no_log(c)); __v; }) +#define readl_relaxed_no_log(c) ({ u32 __v = le32_to_cpu((__force __le32)__raw_readl_no_log(c)); __v; }) +#define readq_relaxed_no_log(c) ({ u64 __v = le64_to_cpu((__force __le64)__raw_readq_no_log(c)); __v; }) + +#define writeb_relaxed_no_log(v, c) ((void)__raw_writeb_no_log((v), (c))) +#define writew_relaxed_no_log(v, c) ((void)__raw_writew_no_log((__force u16)cpu_to_le16(v), (c))) +#define writel_relaxed_no_log(v, c) ((void)__raw_writel_no_log((__force u32)cpu_to_le32(v), (c))) +#define writeq_relaxed_no_log(v, c) ((void)__raw_writeq_no_log((__force u64)cpu_to_le64(v), (c))) + /* * I/O memory access primitives. Reads are ordered relative to any * following Normal memory access. Writes are ordered relative to any prior @@ -142,6 +186,16 @@ #define writel(v,c) ({ __iowmb(); writel_relaxed((v),(c)); }) #define writeq(v,c) ({ __iowmb(); writeq_relaxed((v),(c)); }) +#define readb_no_log(c) ({ u8 __v = readb_relaxed_no_log(c); __iormb(); __v; }) +#define readw_no_log(c) ({ u16 __v = readw_relaxed_no_log(c); __iormb(); __v; }) +#define readl_no_log(c) ({ u32 __v = readl_relaxed_no_log(c); __iormb(); __v; }) +#define readq_no_log(c) ({ u64 __v = readq_relaxed_no_log(c); __iormb(); __v; }) + +#define writeb_no_log(v, c) ({ __iowmb(); writeb_relaxed_no_log((v), (c)); }) +#define writew_no_log(v, c) ({ __iowmb(); writew_relaxed_no_log((v), (c)); }) +#define writel_no_log(v, c) ({ __iowmb(); writel_relaxed_no_log((v), (c)); }) +#define writeq_no_log(v, c) ({ __iowmb(); writeq_relaxed_no_log((v), (c)); }) + /* * I/O port access primitives. */ diff -ruw linux-4.4.115/arch/arm64/include/asm/irq.h linux-4.4.115-fbx/arch/arm64/include/asm/irq.h --- linux-4.4.115/arch/arm64/include/asm/irq.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/irq.h 2019-01-22 16:16:21.539228621 +0100 @@ -1,10 +1,45 @@ #ifndef __ASM_IRQ_H #define __ASM_IRQ_H +#define IRQ_STACK_SIZE THREAD_SIZE +#define IRQ_STACK_START_SP THREAD_START_SP + +#ifndef __ASSEMBLER__ + +#include + #include +#include struct pt_regs; +DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack); + +/* + * The highest address on the stack, and the first to be used. Used to + * find the dummy-stack frame put down by el?_irq() in entry.S, which + * is structured as follows: + * + * ------------ + * | | <- irq_stack_ptr + * top ------------ + * | x19 | <- irq_stack_ptr - 0x08 + * ------------ + * | x29 | <- irq_stack_ptr - 0x10 + * ------------ + * + * where x19 holds a copy of the task stack pointer where the struct pt_regs + * from kernel_entry can be found. + * + */ +#define IRQ_STACK_PTR(cpu) ((unsigned long)per_cpu(irq_stack, cpu) + IRQ_STACK_START_SP) + +/* + * The offset from irq_stack_ptr where entry.S will store the original + * stack pointer. Used by unwind_frame() and dump_backtrace(). + */ +#define IRQ_STACK_TO_TASK_STACK(ptr) (*((unsigned long *)((ptr) - 0x08))) + extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); static inline int nr_legacy_irqs(void) @@ -12,4 +47,17 @@ return 0; } +void arch_trigger_all_cpu_backtrace(void); +#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace + +static inline bool on_irq_stack(unsigned long sp, int cpu) +{ + /* variable names the same as kernel/stacktrace.c */ + unsigned long low = (unsigned long)per_cpu(irq_stack, cpu); + unsigned long high = low + IRQ_STACK_START_SP; + + return (low <= sp && sp <= high); +} + +#endif /* !__ASSEMBLER__ */ #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/kasan.h linux-4.4.115-fbx/arch/arm64/include/asm/kasan.h --- linux-4.4.115/arch/arm64/include/asm/kasan.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/kasan.h 2019-01-22 16:16:21.539228621 +0100 @@ -7,13 +7,14 @@ #include #include +#include /* * KASAN_SHADOW_START: beginning of the kernel virtual addresses. * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/8 of kernel virtual addresses. */ #define KASAN_SHADOW_START (VA_START) -#define KASAN_SHADOW_END (KASAN_SHADOW_START + (1UL << (VA_BITS - 3))) +#define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) /* * This value is used to map an address to the corresponding shadow @@ -28,10 +29,12 @@ #define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << (64 - 3))) void kasan_init(void); +void kasan_copy_shadow(pgd_t *pgdir); asmlinkage void kasan_early_init(void); #else static inline void kasan_init(void) { } +static inline void kasan_copy_shadow(pgd_t *pgdir) { } #endif #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/Kbuild linux-4.4.115-fbx/arch/arm64/include/asm/Kbuild --- linux-4.4.115/arch/arm64/include/asm/Kbuild 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/Kbuild 2019-10-29 09:26:22.993196817 +0100 @@ -2,7 +2,6 @@ generic-y += bug.h generic-y += bugs.h -generic-y += checksum.h generic-y += clkdev.h generic-y += cputime.h generic-y += current.h @@ -14,6 +13,7 @@ generic-y += emergency-restart.h generic-y += errno.h generic-y += ftrace.h +generic-y += hash.h generic-y += hw_irq.h generic-y += ioctl.h generic-y += ioctls.h diff -ruw linux-4.4.115/arch/arm64/include/asm/kernel-pgtable.h linux-4.4.115-fbx/arch/arm64/include/asm/kernel-pgtable.h --- linux-4.4.115/arch/arm64/include/asm/kernel-pgtable.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/kernel-pgtable.h 2019-01-22 16:16:21.539228621 +0100 @@ -19,6 +19,8 @@ #ifndef __ASM_KERNEL_PGTABLE_H #define __ASM_KERNEL_PGTABLE_H +#include +#include /* * The linear mapping and the start of memory are both 2M aligned (per @@ -53,6 +55,12 @@ #define SWAPPER_DIR_SIZE (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE) #define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE) +#ifdef CONFIG_ARM64_SW_TTBR0_PAN +#define RESERVED_TTBR0_SIZE (PAGE_SIZE) +#else +#define RESERVED_TTBR0_SIZE (0) +#endif + /* Initial memory map size */ #if ARM64_SWAPPER_USES_SECTION_MAPS #define SWAPPER_BLOCK_SHIFT SECTION_SHIFT @@ -70,8 +78,16 @@ /* * Initial memory map attributes. */ -#define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) -#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) +#define _SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) +#define _SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) + +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +#define SWAPPER_PTE_FLAGS (_SWAPPER_PTE_FLAGS | PTE_NG) +#define SWAPPER_PMD_FLAGS (_SWAPPER_PMD_FLAGS | PMD_SECT_NG) +#else +#define SWAPPER_PTE_FLAGS _SWAPPER_PTE_FLAGS +#define SWAPPER_PMD_FLAGS _SWAPPER_PMD_FLAGS +#endif #if ARM64_SWAPPER_USES_SECTION_MAPS #define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS) @@ -79,5 +95,31 @@ #define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS) #endif +/* + * To make optimal use of block mappings when laying out the linear + * mapping, round down the base of physical memory to a size that can + * be mapped efficiently, i.e., either PUD_SIZE (4k granule) or PMD_SIZE + * (64k granule), or a multiple that can be mapped using contiguous bits + * in the page tables: 32 * PMD_SIZE (16k granule) + */ +#if defined(CONFIG_ARM64_4K_PAGES) +#define ARM64_MEMSTART_SHIFT PUD_SHIFT +#elif defined(CONFIG_ARM64_16K_PAGES) +#define ARM64_MEMSTART_SHIFT (PMD_SHIFT + 5) +#else +#define ARM64_MEMSTART_SHIFT PMD_SHIFT +#endif + +/* + * sparsemem vmemmap imposes an additional requirement on the alignment of + * memstart_addr, due to the fact that the base of the vmemmap region + * has a direct correspondence, and needs to appear sufficiently aligned + * in the virtual address space. + */ +#if defined(CONFIG_SPARSEMEM_VMEMMAP) && ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS +#define ARM64_MEMSTART_ALIGN (1UL << SECTION_SIZE_BITS) +#else +#define ARM64_MEMSTART_ALIGN (1UL << ARM64_MEMSTART_SHIFT) +#endif #endif /* __ASM_KERNEL_PGTABLE_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/kvm_arm.h linux-4.4.115-fbx/arch/arm64/include/asm/kvm_arm.h --- linux-4.4.115/arch/arm64/include/asm/kvm_arm.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/kvm_arm.h 2019-10-29 09:26:22.997196856 +0100 @@ -83,17 +83,6 @@ #define HCR_INT_OVERRIDE (HCR_FMO | HCR_IMO) -/* Hyp System Control Register (SCTLR_EL2) bits */ -#define SCTLR_EL2_EE (1 << 25) -#define SCTLR_EL2_WXN (1 << 19) -#define SCTLR_EL2_I (1 << 12) -#define SCTLR_EL2_SA (1 << 3) -#define SCTLR_EL2_C (1 << 2) -#define SCTLR_EL2_A (1 << 1) -#define SCTLR_EL2_M 1 -#define SCTLR_EL2_FLAGS (SCTLR_EL2_M | SCTLR_EL2_A | SCTLR_EL2_C | \ - SCTLR_EL2_SA | SCTLR_EL2_I) - /* TCR_EL2 Registers bits */ #define TCR_EL2_RES1 ((1 << 31) | (1 << 23)) #define TCR_EL2_TBI (1 << 20) @@ -123,6 +112,7 @@ #define VTCR_EL2_SL0_LVL1 (1 << 6) #define VTCR_EL2_T0SZ_MASK 0x3f #define VTCR_EL2_T0SZ_40B 24 +#define VTCR_EL2_VS 19 /* * We configure the Stage-2 page tables to always restrict the IPA space to be @@ -166,7 +156,7 @@ #define VTTBR_BADDR_MASK (((UL(1) << (PHYS_MASK_SHIFT - VTTBR_X)) - 1) << VTTBR_X) #define VTTBR_VMID_SHIFT (UL(48)) -#define VTTBR_VMID_MASK (UL(0xFF) << VTTBR_VMID_SHIFT) +#define VTTBR_VMID_MASK(size) (_AT(u64, (1 << size) - 1) << VTTBR_VMID_SHIFT) /* Hyp System Trap Register */ #define HSTR_EL2_T(x) (1 << x) diff -ruw linux-4.4.115/arch/arm64/include/asm/kvm_asm.h linux-4.4.115-fbx/arch/arm64/include/asm/kvm_asm.h --- linux-4.4.115/arch/arm64/include/asm/kvm_asm.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/kvm_asm.h 2019-01-22 16:16:21.539228621 +0100 @@ -20,94 +20,38 @@ #include -/* - * 0 is reserved as an invalid value. - * Order *must* be kept in sync with the hyp switch code. - */ -#define MPIDR_EL1 1 /* MultiProcessor Affinity Register */ -#define CSSELR_EL1 2 /* Cache Size Selection Register */ -#define SCTLR_EL1 3 /* System Control Register */ -#define ACTLR_EL1 4 /* Auxiliary Control Register */ -#define CPACR_EL1 5 /* Coprocessor Access Control */ -#define TTBR0_EL1 6 /* Translation Table Base Register 0 */ -#define TTBR1_EL1 7 /* Translation Table Base Register 1 */ -#define TCR_EL1 8 /* Translation Control Register */ -#define ESR_EL1 9 /* Exception Syndrome Register */ -#define AFSR0_EL1 10 /* Auxilary Fault Status Register 0 */ -#define AFSR1_EL1 11 /* Auxilary Fault Status Register 1 */ -#define FAR_EL1 12 /* Fault Address Register */ -#define MAIR_EL1 13 /* Memory Attribute Indirection Register */ -#define VBAR_EL1 14 /* Vector Base Address Register */ -#define CONTEXTIDR_EL1 15 /* Context ID Register */ -#define TPIDR_EL0 16 /* Thread ID, User R/W */ -#define TPIDRRO_EL0 17 /* Thread ID, User R/O */ -#define TPIDR_EL1 18 /* Thread ID, Privileged */ -#define AMAIR_EL1 19 /* Aux Memory Attribute Indirection Register */ -#define CNTKCTL_EL1 20 /* Timer Control Register (EL1) */ -#define PAR_EL1 21 /* Physical Address Register */ -#define MDSCR_EL1 22 /* Monitor Debug System Control Register */ -#define MDCCINT_EL1 23 /* Monitor Debug Comms Channel Interrupt Enable Reg */ - -/* 32bit specific registers. Keep them at the end of the range */ -#define DACR32_EL2 24 /* Domain Access Control Register */ -#define IFSR32_EL2 25 /* Instruction Fault Status Register */ -#define FPEXC32_EL2 26 /* Floating-Point Exception Control Register */ -#define DBGVCR32_EL2 27 /* Debug Vector Catch Register */ -#define NR_SYS_REGS 28 - -/* 32bit mapping */ -#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */ -#define c0_CSSELR (CSSELR_EL1 * 2)/* Cache Size Selection Register */ -#define c1_SCTLR (SCTLR_EL1 * 2) /* System Control Register */ -#define c1_ACTLR (ACTLR_EL1 * 2) /* Auxiliary Control Register */ -#define c1_CPACR (CPACR_EL1 * 2) /* Coprocessor Access Control */ -#define c2_TTBR0 (TTBR0_EL1 * 2) /* Translation Table Base Register 0 */ -#define c2_TTBR0_high (c2_TTBR0 + 1) /* TTBR0 top 32 bits */ -#define c2_TTBR1 (TTBR1_EL1 * 2) /* Translation Table Base Register 1 */ -#define c2_TTBR1_high (c2_TTBR1 + 1) /* TTBR1 top 32 bits */ -#define c2_TTBCR (TCR_EL1 * 2) /* Translation Table Base Control R. */ -#define c3_DACR (DACR32_EL2 * 2)/* Domain Access Control Register */ -#define c5_DFSR (ESR_EL1 * 2) /* Data Fault Status Register */ -#define c5_IFSR (IFSR32_EL2 * 2)/* Instruction Fault Status Register */ -#define c5_ADFSR (AFSR0_EL1 * 2) /* Auxiliary Data Fault Status R */ -#define c5_AIFSR (AFSR1_EL1 * 2) /* Auxiliary Instr Fault Status R */ -#define c6_DFAR (FAR_EL1 * 2) /* Data Fault Address Register */ -#define c6_IFAR (c6_DFAR + 1) /* Instruction Fault Address Register */ -#define c7_PAR (PAR_EL1 * 2) /* Physical Address Register */ -#define c7_PAR_high (c7_PAR + 1) /* PAR top 32 bits */ -#define c10_PRRR (MAIR_EL1 * 2) /* Primary Region Remap Register */ -#define c10_NMRR (c10_PRRR + 1) /* Normal Memory Remap Register */ -#define c12_VBAR (VBAR_EL1 * 2) /* Vector Base Address Register */ -#define c13_CID (CONTEXTIDR_EL1 * 2) /* Context ID Register */ -#define c13_TID_URW (TPIDR_EL0 * 2) /* Thread ID, User R/W */ -#define c13_TID_URO (TPIDRRO_EL0 * 2)/* Thread ID, User R/O */ -#define c13_TID_PRIV (TPIDR_EL1 * 2) /* Thread ID, Privileged */ -#define c10_AMAIR0 (AMAIR_EL1 * 2) /* Aux Memory Attr Indirection Reg */ -#define c10_AMAIR1 (c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */ -#define c14_CNTKCTL (CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */ - -#define cp14_DBGDSCRext (MDSCR_EL1 * 2) -#define cp14_DBGBCR0 (DBGBCR0_EL1 * 2) -#define cp14_DBGBVR0 (DBGBVR0_EL1 * 2) -#define cp14_DBGBXVR0 (cp14_DBGBVR0 + 1) -#define cp14_DBGWCR0 (DBGWCR0_EL1 * 2) -#define cp14_DBGWVR0 (DBGWVR0_EL1 * 2) -#define cp14_DBGDCCINT (MDCCINT_EL1 * 2) - -#define NR_COPRO_REGS (NR_SYS_REGS * 2) - #define ARM_EXCEPTION_IRQ 0 #define ARM_EXCEPTION_TRAP 1 +/* The hyp-stub will return this for any kvm_call_hyp() call */ +#define ARM_EXCEPTION_HYP_GONE 2 #define KVM_ARM64_DEBUG_DIRTY_SHIFT 0 #define KVM_ARM64_DEBUG_DIRTY (1 << KVM_ARM64_DEBUG_DIRTY_SHIFT) +#define kvm_ksym_ref(sym) phys_to_virt((u64)&sym - kimage_voffset) + #ifndef __ASSEMBLY__ +#if __GNUC__ > 4 +#define kvm_ksym_shift (PAGE_OFFSET - KIMAGE_VADDR) +#else +/* + * GCC versions 4.9 and older will fold the constant below into the addend of + * the reference to 'sym' above if kvm_ksym_shift is declared static or if the + * constant is used directly. However, since we use the small code model for + * the core kernel, the reference to 'sym' will be emitted as a adrp/add pair, + * with a +/- 4 GB range, resulting in linker relocation errors if the shift + * is sufficiently large. So prevent the compiler from folding the shift into + * the addend, by making the shift a variable with external linkage. + */ +__weak u64 kvm_ksym_shift = PAGE_OFFSET - KIMAGE_VADDR; +#endif + struct kvm; struct kvm_vcpu; extern char __kvm_hyp_init[]; extern char __kvm_hyp_init_end[]; +extern char __kvm_hyp_reset[]; extern char __kvm_hyp_vector[]; diff -ruw linux-4.4.115/arch/arm64/include/asm/kvm_host.h linux-4.4.115-fbx/arch/arm64/include/asm/kvm_host.h --- linux-4.4.115/arch/arm64/include/asm/kvm_host.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/kvm_host.h 2019-01-22 16:16:21.539228621 +0100 @@ -25,7 +25,6 @@ #include #include #include -#include #include #define __KVM_HAVE_ARCH_INTC_INITIALIZED @@ -45,6 +44,7 @@ int __attribute_const__ kvm_target_cpu(void); int kvm_reset_vcpu(struct kvm_vcpu *vcpu); int kvm_arch_dev_ioctl_check_extension(long ext); +phys_addr_t kvm_hyp_reset_entry(void); struct kvm_arch { /* The VMID generation used for the virt. memory system */ @@ -85,6 +85,86 @@ u64 hpfar_el2; /* Hyp IPA Fault Address Register */ }; +/* + * 0 is reserved as an invalid value. + * Order should be kept in sync with the save/restore code. + */ +enum vcpu_sysreg { + __INVALID_SYSREG__, + MPIDR_EL1, /* MultiProcessor Affinity Register */ + CSSELR_EL1, /* Cache Size Selection Register */ + SCTLR_EL1, /* System Control Register */ + ACTLR_EL1, /* Auxiliary Control Register */ + CPACR_EL1, /* Coprocessor Access Control */ + TTBR0_EL1, /* Translation Table Base Register 0 */ + TTBR1_EL1, /* Translation Table Base Register 1 */ + TCR_EL1, /* Translation Control Register */ + ESR_EL1, /* Exception Syndrome Register */ + AFSR0_EL1, /* Auxilary Fault Status Register 0 */ + AFSR1_EL1, /* Auxilary Fault Status Register 1 */ + FAR_EL1, /* Fault Address Register */ + MAIR_EL1, /* Memory Attribute Indirection Register */ + VBAR_EL1, /* Vector Base Address Register */ + CONTEXTIDR_EL1, /* Context ID Register */ + TPIDR_EL0, /* Thread ID, User R/W */ + TPIDRRO_EL0, /* Thread ID, User R/O */ + TPIDR_EL1, /* Thread ID, Privileged */ + AMAIR_EL1, /* Aux Memory Attribute Indirection Register */ + CNTKCTL_EL1, /* Timer Control Register (EL1) */ + PAR_EL1, /* Physical Address Register */ + MDSCR_EL1, /* Monitor Debug System Control Register */ + MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */ + + /* 32bit specific registers. Keep them at the end of the range */ + DACR32_EL2, /* Domain Access Control Register */ + IFSR32_EL2, /* Instruction Fault Status Register */ + FPEXC32_EL2, /* Floating-Point Exception Control Register */ + DBGVCR32_EL2, /* Debug Vector Catch Register */ + + NR_SYS_REGS /* Nothing after this line! */ +}; + +/* 32bit mapping */ +#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */ +#define c0_CSSELR (CSSELR_EL1 * 2)/* Cache Size Selection Register */ +#define c1_SCTLR (SCTLR_EL1 * 2) /* System Control Register */ +#define c1_ACTLR (ACTLR_EL1 * 2) /* Auxiliary Control Register */ +#define c1_CPACR (CPACR_EL1 * 2) /* Coprocessor Access Control */ +#define c2_TTBR0 (TTBR0_EL1 * 2) /* Translation Table Base Register 0 */ +#define c2_TTBR0_high (c2_TTBR0 + 1) /* TTBR0 top 32 bits */ +#define c2_TTBR1 (TTBR1_EL1 * 2) /* Translation Table Base Register 1 */ +#define c2_TTBR1_high (c2_TTBR1 + 1) /* TTBR1 top 32 bits */ +#define c2_TTBCR (TCR_EL1 * 2) /* Translation Table Base Control R. */ +#define c3_DACR (DACR32_EL2 * 2)/* Domain Access Control Register */ +#define c5_DFSR (ESR_EL1 * 2) /* Data Fault Status Register */ +#define c5_IFSR (IFSR32_EL2 * 2)/* Instruction Fault Status Register */ +#define c5_ADFSR (AFSR0_EL1 * 2) /* Auxiliary Data Fault Status R */ +#define c5_AIFSR (AFSR1_EL1 * 2) /* Auxiliary Instr Fault Status R */ +#define c6_DFAR (FAR_EL1 * 2) /* Data Fault Address Register */ +#define c6_IFAR (c6_DFAR + 1) /* Instruction Fault Address Register */ +#define c7_PAR (PAR_EL1 * 2) /* Physical Address Register */ +#define c7_PAR_high (c7_PAR + 1) /* PAR top 32 bits */ +#define c10_PRRR (MAIR_EL1 * 2) /* Primary Region Remap Register */ +#define c10_NMRR (c10_PRRR + 1) /* Normal Memory Remap Register */ +#define c12_VBAR (VBAR_EL1 * 2) /* Vector Base Address Register */ +#define c13_CID (CONTEXTIDR_EL1 * 2) /* Context ID Register */ +#define c13_TID_URW (TPIDR_EL0 * 2) /* Thread ID, User R/W */ +#define c13_TID_URO (TPIDRRO_EL0 * 2)/* Thread ID, User R/O */ +#define c13_TID_PRIV (TPIDR_EL1 * 2) /* Thread ID, Privileged */ +#define c10_AMAIR0 (AMAIR_EL1 * 2) /* Aux Memory Attr Indirection Reg */ +#define c10_AMAIR1 (c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */ +#define c14_CNTKCTL (CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */ + +#define cp14_DBGDSCRext (MDSCR_EL1 * 2) +#define cp14_DBGBCR0 (DBGBCR0_EL1 * 2) +#define cp14_DBGBVR0 (DBGBVR0_EL1 * 2) +#define cp14_DBGBXVR0 (cp14_DBGBVR0 + 1) +#define cp14_DBGWCR0 (DBGWCR0_EL1 * 2) +#define cp14_DBGWVR0 (DBGWVR0_EL1 * 2) +#define cp14_DBGDCCINT (MDCCINT_EL1 * 2) + +#define NR_COPRO_REGS (NR_SYS_REGS * 2) + struct kvm_cpu_context { struct kvm_regs gp_regs; union { @@ -222,7 +302,7 @@ struct kvm_vcpu *kvm_arm_get_running_vcpu(void); struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void); -u64 kvm_call_hyp(void *hypfn, ...); +u64 __kvm_call_hyp(void *hypfn, ...); void force_vm_exit(const cpumask_t *mask); void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot); @@ -243,11 +323,25 @@ * Call initialization code, and switch to the full blown * HYP code. */ - kvm_call_hyp((void *)boot_pgd_ptr, pgd_ptr, + __kvm_call_hyp((void *)boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr); } -static inline void kvm_arch_hardware_disable(void) {} +static inline void __cpu_init_stage2(void) +{ +} + +static inline void __cpu_reset_hyp_mode(phys_addr_t boot_pgd_ptr, + phys_addr_t phys_idmap_start) +{ + /* + * Call reset code, and switch back to stub hyp vectors. + * Uses __kvm_call_hyp() to avoid kaslr's kvm_ksym_ref() translation. + */ + __kvm_call_hyp((void *)kvm_hyp_reset_entry(), + boot_pgd_ptr, phys_idmap_start); +} + static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} @@ -258,4 +352,6 @@ void kvm_arm_clear_debug(struct kvm_vcpu *vcpu); void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu); +#define kvm_call_hyp(f, ...) __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__) + #endif /* __ARM64_KVM_HOST_H__ */ diff -ruw linux-4.4.115/arch/arm64/include/asm/kvm_mmio.h linux-4.4.115-fbx/arch/arm64/include/asm/kvm_mmio.h --- linux-4.4.115/arch/arm64/include/asm/kvm_mmio.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/kvm_mmio.h 2019-01-22 16:16:21.539228621 +0100 @@ -19,7 +19,6 @@ #define __ARM64_KVM_MMIO_H__ #include -#include #include /* diff -ruw linux-4.4.115/arch/arm64/include/asm/lse.h linux-4.4.115-fbx/arch/arm64/include/asm/lse.h --- linux-4.4.115/arch/arm64/include/asm/lse.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/lse.h 2019-01-22 16:16:21.539228621 +0100 @@ -26,6 +26,7 @@ /* Macro for constructing calls to out-of-line ll/sc atomics */ #define __LL_SC_CALL(op) "bl\t" __stringify(__LL_SC_PREFIX(op)) "\n" +#define __LL_SC_CLOBBERS "x16", "x17", "x30" /* In-line patching at runtime */ #define ARM64_LSE_ATOMIC_INSN(llsc, lse) \ diff -ruw linux-4.4.115/arch/arm64/include/asm/memory.h linux-4.4.115-fbx/arch/arm64/include/asm/memory.h --- linux-4.4.115/arch/arm64/include/asm/memory.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/memory.h 2019-10-29 09:26:22.997196856 +0100 @@ -24,6 +24,7 @@ #include #include #include +#include #include /* @@ -45,15 +46,15 @@ * VA_START - the first kernel virtual address. * TASK_SIZE - the maximum size of a user space task. * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area. - * The module space lives between the addresses given by TASK_SIZE - * and PAGE_OFFSET - it must be within 128MB of the kernel text. */ #define VA_BITS (CONFIG_ARM64_VA_BITS) #define VA_START (UL(0xffffffffffffffff) << VA_BITS) #define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) -#define MODULES_END (PAGE_OFFSET) -#define MODULES_VADDR (MODULES_END - SZ_64M) -#define PCI_IO_END (MODULES_VADDR - SZ_2M) +#define KIMAGE_VADDR (MODULES_END) +#define MODULES_END (MODULES_VADDR + MODULES_VSIZE) +#define MODULES_VADDR (VA_START + KASAN_SHADOW_SIZE) +#define MODULES_VSIZE (SZ_128M) +#define PCI_IO_END (PAGE_OFFSET - SZ_2M) #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) #define FIXADDR_TOP (PCI_IO_START - SZ_2M) #define TASK_SIZE_64 (UL(1) << VA_BITS) @@ -70,13 +71,31 @@ #define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4)) +#define KERNEL_START _text +#define KERNEL_END _end + +/* + * The size of the KASAN shadow region. This should be 1/8th of the + * size of the entire kernel virtual address space. + */ +#ifdef CONFIG_KASAN +#define KASAN_SHADOW_SIZE (UL(1) << (VA_BITS - 3)) +#else +#define KASAN_SHADOW_SIZE (0) +#endif + /* * Physical vs virtual RAM address space conversion. These are * private definitions which should NOT be used outside memory.h * files. Use virt_to_phys/phys_to_virt/__pa/__va instead. */ -#define __virt_to_phys(x) (((phys_addr_t)(x) - PAGE_OFFSET + PHYS_OFFSET)) -#define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET + PAGE_OFFSET)) +#define __virt_to_phys(x) ({ \ + phys_addr_t __x = (phys_addr_t)(x); \ + __x & BIT(VA_BITS - 1) ? (__x & ~PAGE_OFFSET) + PHYS_OFFSET : \ + (__x - kimage_voffset); }) + +#define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET) | PAGE_OFFSET) +#define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset)) /* * Convert a page to/from a physical address @@ -100,19 +119,45 @@ #define MT_S2_NORMAL 0xf #define MT_S2_DEVICE_nGnRE 0x1 +#ifdef CONFIG_ARM64_4K_PAGES +#define IOREMAP_MAX_ORDER (PUD_SHIFT) +#else +#define IOREMAP_MAX_ORDER (PMD_SHIFT) +#endif + +#ifdef CONFIG_BLK_DEV_INITRD +#define __early_init_dt_declare_initrd(__start, __end) \ + do { \ + initrd_start = (__start); \ + initrd_end = (__end); \ + } while (0) +#endif + #ifndef __ASSEMBLY__ -extern phys_addr_t memstart_addr; +#include +#include + +extern s64 memstart_addr; /* PHYS_OFFSET - the physical address of the start of memory. */ -#define PHYS_OFFSET ({ memstart_addr; }) +#define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; }) + +/* the virtual base of the kernel image (minus TEXT_OFFSET) */ +extern u64 kimage_vaddr; + +/* the offset between the kernel virtual and physical mappings */ +extern u64 kimage_voffset; + +static inline unsigned long kaslr_offset(void) +{ + return kimage_vaddr - KIMAGE_VADDR; +} /* - * The maximum physical address that the linear direct mapping - * of system RAM can cover. (PAGE_OFFSET can be interpreted as - * a 2's complement signed quantity and negated to derive the - * maximum size of the linear mapping.) + * Allow all memory at the discovery stage. We will clip it later. */ -#define MAX_MEMBLOCK_ADDR ({ memstart_addr - PAGE_OFFSET - 1; }) +#define MIN_MEMBLOCK_ADDR 0 +#define MAX_MEMBLOCK_ADDR U64_MAX /* * PFNs are used to describe any physical page; this means @@ -148,6 +193,7 @@ #define __va(x) ((void *)__phys_to_virt((phys_addr_t)(x))) #define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) #define virt_to_pfn(x) __phys_to_pfn(__virt_to_phys(x)) +#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) /* * virt_to_page(k) convert a _valid_ virtual address to struct page * @@ -156,7 +202,11 @@ #define ARCH_PFN_OFFSET ((unsigned long)PHYS_PFN_OFFSET) #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) -#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) +#define _virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) + +#define _virt_addr_is_linear(kaddr) (((u64)(kaddr)) >= PAGE_OFFSET) +#define virt_addr_valid(kaddr) (_virt_addr_is_linear(kaddr) && \ + _virt_addr_valid(kaddr)) #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/mmu_context.h linux-4.4.115-fbx/arch/arm64/include/asm/mmu_context.h --- linux-4.4.115/arch/arm64/include/asm/mmu_context.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/mmu_context.h 2019-01-22 16:16:21.539228621 +0100 @@ -23,23 +23,30 @@ #include #include +#include #include #include #include #include +#include +#include #ifdef CONFIG_PID_IN_CONTEXTIDR static inline void contextidr_thread_switch(struct task_struct *next) { + pid_t pid = task_pid_nr(next); asm( " msr contextidr_el1, %0\n" " isb" : - : "r" (task_pid_nr(next))); + : "r" (pid)); + uncached_logk(LOGK_CTXID, (void *)(u64)pid); + } #else static inline void contextidr_thread_switch(struct task_struct *next) { + uncached_logk(LOGK_CTXID, (void *)(u64)task_pid_nr(next)); } #endif @@ -48,7 +55,7 @@ */ static inline void cpu_set_reserved_ttbr0(void) { - unsigned long ttbr = page_to_phys(empty_zero_page); + unsigned long ttbr = __pa_symbol(empty_zero_page); asm( " msr ttbr0_el1, %0 // set TTBR0\n" @@ -57,6 +64,13 @@ : "r" (ttbr)); } +static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) +{ + BUG_ON(pgd == swapper_pg_dir); + cpu_set_reserved_ttbr0(); + cpu_do_switch_mm(virt_to_phys(pgd),mm); +} + /* * TCR.T0SZ value to use when the ID map is active. Usually equals * TCR_T0SZ(VA_BITS), unless system RAM is positioned very high in @@ -73,7 +87,7 @@ /* * Set TCR.T0SZ to its default value (based on VA_BITS) */ -static inline void cpu_set_default_tcr_t0sz(void) +static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) { unsigned long tcr; @@ -86,7 +100,62 @@ " msr tcr_el1, %0 ;" " isb" : "=&r" (tcr) - : "r"(TCR_T0SZ(VA_BITS)), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH)); + : "r"(t0sz), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH)); +} + +#define cpu_set_default_tcr_t0sz() __cpu_set_tcr_t0sz(TCR_T0SZ(VA_BITS)) +#define cpu_set_idmap_tcr_t0sz() __cpu_set_tcr_t0sz(idmap_t0sz) + +/* + * Remove the idmap from TTBR0_EL1 and install the pgd of the active mm. + * + * The idmap lives in the same VA range as userspace, but uses global entries + * and may use a different TCR_EL1.T0SZ. To avoid issues resulting from + * speculative TLB fetches, we must temporarily install the reserved page + * tables while we invalidate the TLBs and set up the correct TCR_EL1.T0SZ. + * + * If current is a not a user task, the mm covers the TTBR1_EL1 page tables, + * which should not be installed in TTBR0_EL1. In this case we can leave the + * reserved page tables in place. + */ +static inline void cpu_uninstall_idmap(void) +{ + struct mm_struct *mm = current->active_mm; + + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + cpu_set_default_tcr_t0sz(); + + if (mm != &init_mm && !system_uses_ttbr0_pan()) + cpu_switch_mm(mm->pgd, mm); +} + +static inline void cpu_install_idmap(void) +{ + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + cpu_set_idmap_tcr_t0sz(); + + cpu_switch_mm(lm_alias(idmap_pg_dir), &init_mm); +} + +/* + * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD, + * avoiding the possibility of conflicting TLB entries being allocated. + */ +static inline void cpu_replace_ttbr1(pgd_t *pgd) +{ + typedef void (ttbr_replace_func)(phys_addr_t); + extern ttbr_replace_func idmap_cpu_replace_ttbr1; + ttbr_replace_func *replace_phys; + + phys_addr_t pgd_phys = virt_to_phys(pgd); + + replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); + + cpu_install_idmap(); + replace_phys(pgd_phys); + cpu_uninstall_idmap(); } /* @@ -103,35 +172,43 @@ #define init_new_context(tsk,mm) ({ atomic64_set(&(mm)->context.id, 0); 0; }) -/* - * This is called when "tsk" is about to enter lazy TLB mode. - * - * mm: describes the currently active mm context - * tsk: task which is entering lazy tlb - * cpu: cpu number which is entering lazy tlb - * - * tsk->mm will be NULL - */ -static inline void -enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) +#ifdef CONFIG_ARM64_SW_TTBR0_PAN +static inline void update_saved_ttbr0(struct task_struct *tsk, + struct mm_struct *mm) +{ + u64 ttbr; + + if (!system_uses_ttbr0_pan()) + return; + + if (mm == &init_mm) + ttbr = __pa_symbol(empty_zero_page); + else + ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48; + + WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr); +} +#else +static inline void update_saved_ttbr0(struct task_struct *tsk, + struct mm_struct *mm) { } +#endif +static inline void +enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) +{ /* - * This is the actual mm switch as far as the scheduler - * is concerned. No registers are touched. We avoid - * calling the CPU specific function when the mm hasn't - * actually changed. + * We don't actually care about the ttbr0 mapping, so point it at the + * zero page. */ -static inline void -switch_mm(struct mm_struct *prev, struct mm_struct *next, - struct task_struct *tsk) + update_saved_ttbr0(tsk, &init_mm); +} + +static inline void __switch_mm(struct mm_struct *next) { unsigned int cpu = smp_processor_id(); - if (prev == next) - return; - /* * init_mm.pgd does not contain any user mappings and it is always * active for kernel addresses in TTBR1. Just set the reserved TTBR0. @@ -144,7 +221,25 @@ check_and_switch_context(next, cpu); } +static inline void +switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + if (prev != next) + __switch_mm(next); + + /* + * Update the saved TTBR0_EL1 of the scheduled-in task as the previous + * value may have not been initialised yet (activate_mm caller) or the + * ASID has changed since the last run (following the context switch + * of another thread of the same process). + */ + update_saved_ttbr0(tsk, next); +} + #define deactivate_mm(tsk,mm) do { } while (0) -#define activate_mm(prev,next) switch_mm(prev, next, NULL) +#define activate_mm(prev,next) switch_mm(prev, next, current) + +void post_ttbr_update_workaround(void); #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/mmu.h linux-4.4.115-fbx/arch/arm64/include/asm/mmu.h --- linux-4.4.115/arch/arm64/include/asm/mmu.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/mmu.h 2019-01-22 16:16:21.539228621 +0100 @@ -16,6 +16,15 @@ #ifndef __ASM_MMU_H #define __ASM_MMU_H +#define USER_ASID_FLAG (UL(1) << 48) +#define TTBR_ASID_MASK (UL(0xffff) << 48) + +#ifndef __ASSEMBLY__ +#include + +#include +#include + typedef struct { atomic64_t id; void *vdso; @@ -28,6 +37,49 @@ */ #define ASID(mm) ((mm)->context.id.counter & 0xffff) +static inline bool arm64_kernel_unmapped_at_el0(void) +{ + return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) && + cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0); +} + +typedef void (*bp_hardening_cb_t)(void); + +struct bp_hardening_data { + int hyp_vectors_slot; + bp_hardening_cb_t fn; +}; + +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR +extern char __bp_harden_hyp_vecs_start[], __bp_harden_hyp_vecs_end[]; + +DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); + +static inline struct bp_hardening_data *arm64_get_bp_hardening_data(void) +{ + return this_cpu_ptr(&bp_hardening_data); +} + +static inline void arm64_apply_bp_hardening(void) +{ + struct bp_hardening_data *d; + + if (!cpus_have_cap(ARM64_HARDEN_BRANCH_PREDICTOR)) + return; + + d = arm64_get_bp_hardening_data(); + if (d->fn) + d->fn(); +} +#else +static inline struct bp_hardening_data *arm64_get_bp_hardening_data(void) +{ + return NULL; +} + +static inline void arm64_apply_bp_hardening(void) { } +#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */ + extern void paging_init(void); extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); extern void init_mem_pgprot(void); @@ -35,5 +87,13 @@ unsigned long virt, phys_addr_t size, pgprot_t prot); extern void *fixmap_remap_fdt(phys_addr_t dt_phys); +#ifdef CONFIG_MEMORY_HOTPLUG +extern void hotplug_paging(phys_addr_t start, phys_addr_t size); +#ifdef CONFIG_MEMORY_HOTREMOVE +extern void remove_pagetable(unsigned long start, + unsigned long end, bool direct); +#endif +#endif +#endif /* !__ASSEMBLY__ */ #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/module.h linux-4.4.115-fbx/arch/arm64/include/asm/module.h --- linux-4.4.115/arch/arm64/include/asm/module.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/module.h 2019-01-22 16:16:21.539228621 +0100 @@ -17,7 +17,29 @@ #define __ASM_MODULE_H #include +#include #define MODULE_ARCH_VERMAGIC "aarch64" +#ifdef CONFIG_ARM64_MODULE_PLTS +struct mod_arch_specific { + struct elf64_shdr *plt; + int plt_num_entries; + int plt_max_entries; +}; +#endif + +u64 module_emit_plt_entry(struct module *mod, const Elf64_Rela *rela, + Elf64_Sym *sym); + +#ifdef CONFIG_RANDOMIZE_BASE +#ifdef CONFIG_MODVERSIONS +#define ARCH_RELOCATES_KCRCTAB +#define reloc_start (kimage_vaddr - KIMAGE_VADDR) +#endif +extern u64 module_alloc_base; +#else +#define module_alloc_base ((u64)_etext - MODULES_VSIZE) +#endif + #endif /* __ASM_MODULE_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/page.h linux-4.4.115-fbx/arch/arm64/include/asm/page.h --- linux-4.4.115/arch/arm64/include/asm/page.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/page.h 2019-01-22 16:16:21.539228621 +0100 @@ -19,6 +19,8 @@ #ifndef __ASM_PAGE_H #define __ASM_PAGE_H +#include + /* PAGE_SHIFT determines the page size */ /* CONT_SHIFT determines the number of pages which can be tracked together */ #ifdef CONFIG_ARM64_64K_PAGES diff -ruw linux-4.4.115/arch/arm64/include/asm/pci.h linux-4.4.115-fbx/arch/arm64/include/asm/pci.h --- linux-4.4.115/arch/arm64/include/asm/pci.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/pci.h 2019-01-22 16:16:21.543228657 +0100 @@ -41,3 +41,8 @@ #endif /* __KERNEL__ */ #endif /* __ASM_PCI_H */ + +#ifdef CONFIG_PCI_MSM +#define arch_setup_msi_irqs arch_setup_msi_irqs +#define arch_teardown_msi_irqs arch_teardown_msi_irqs +#endif diff -ruw linux-4.4.115/arch/arm64/include/asm/percpu.h linux-4.4.115-fbx/arch/arm64/include/asm/percpu.h --- linux-4.4.115/arch/arm64/include/asm/percpu.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/percpu.h 2019-10-29 09:26:22.997196856 +0100 @@ -16,6 +16,8 @@ #ifndef __ASM_PERCPU_H #define __ASM_PERCPU_H +#include + static inline void set_my_cpu_offset(unsigned long off) { asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory"); diff -ruw linux-4.4.115/arch/arm64/include/asm/perf_event.h linux-4.4.115-fbx/arch/arm64/include/asm/perf_event.h --- linux-4.4.115/arch/arm64/include/asm/perf_event.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/perf_event.h 2019-01-22 16:16:21.543228657 +0100 @@ -17,6 +17,8 @@ #ifndef __ASM_PERF_EVENT_H #define __ASM_PERF_EVENT_H +#include + #ifdef CONFIG_PERF_EVENTS struct pt_regs; extern unsigned long perf_instruction_pointer(struct pt_regs *regs); @@ -31,4 +33,91 @@ (regs)->pstate = PSR_MODE_EL1h; \ } +static inline u32 armv8pmu_pmcr_read_reg(void) +{ + u32 val; + + asm volatile("mrs %0, pmcr_el0" : "=r" (val)); + return val; +} + +static inline u32 armv8pmu_pmccntr_read_reg(void) +{ + u32 val; + + asm volatile("mrs %0, pmccntr_el0" : "=r" (val)); + return val; +} + +static inline u32 armv8pmu_pmxevcntr_read_reg(void) +{ + u32 val; + + asm volatile("mrs %0, pmxevcntr_el0" : "=r" (val)); + return val; +} + +static inline u32 armv8pmu_pmovsclr_read_reg(void) +{ + u32 val; + + asm volatile("mrs %0, pmovsclr_el0" : "=r" (val)); + return val; +} + +static inline void armv8pmu_pmcr_write_reg(u32 val) +{ + asm volatile("msr pmcr_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmselr_write_reg(u32 val) +{ + asm volatile("msr pmselr_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmccntr_write_reg(u32 val) +{ + asm volatile("msr pmccntr_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmxevcntr_write_reg(u32 val) +{ + asm volatile("msr pmxevcntr_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmxevtyper_write_reg(u32 val) +{ + asm volatile("msr pmxevtyper_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmcntenset_write_reg(u32 val) +{ + asm volatile("msr pmcntenset_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmcntenclr_write_reg(u32 val) +{ + asm volatile("msr pmcntenclr_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmintenset_write_reg(u32 val) +{ + asm volatile("msr pmintenset_el1, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmintenclr_write_reg(u32 val) +{ + asm volatile("msr pmintenclr_el1, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmovsclr_write_reg(u32 val) +{ + asm volatile("msr pmovsclr_el0, %0" :: "r" (val)); +} + +static inline void armv8pmu_pmuserenr_write_reg(u32 val) +{ + asm volatile("msr pmuserenr_el0, %0" :: "r" (val)); +} + #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/pgalloc.h linux-4.4.115-fbx/arch/arm64/include/asm/pgalloc.h --- linux-4.4.115/arch/arm64/include/asm/pgalloc.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/pgalloc.h 2019-01-22 16:16:21.543228657 +0100 @@ -42,11 +42,20 @@ free_page((unsigned long)pmd); } -static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot) { - set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE)); + set_pud(pud, __pud(pmd | prot)); } +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + __pud_populate(pud, __pa(pmd), PMD_TYPE_TABLE); +} +#else +static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot) +{ + BUILD_BUG(); +} #endif /* CONFIG_PGTABLE_LEVELS > 2 */ #if CONFIG_PGTABLE_LEVELS > 3 @@ -62,11 +71,20 @@ free_page((unsigned long)pud); } -static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) +static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot) { - set_pgd(pgd, __pgd(__pa(pud) | PUD_TYPE_TABLE)); + set_pgd(pgdp, __pgd(pud | prot)); } +static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) +{ + __pgd_populate(pgd, __pa(pud), PUD_TYPE_TABLE); +} +#else +static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot) +{ + BUILD_BUG(); +} #endif /* CONFIG_PGTABLE_LEVELS > 3 */ extern pgd_t *pgd_alloc(struct mm_struct *mm); diff -ruw linux-4.4.115/arch/arm64/include/asm/pgtable.h linux-4.4.115-fbx/arch/arm64/include/asm/pgtable.h --- linux-4.4.115/arch/arm64/include/asm/pgtable.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/pgtable.h 2019-10-29 09:26:22.997196856 +0100 @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -36,19 +37,13 @@ * * VMEMAP_SIZE: allows the whole linear region to be covered by a struct page array * (rounded up to PUD_SIZE). - * VMALLOC_START: beginning of the kernel VA space + * VMALLOC_START: beginning of the kernel vmalloc space * VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space, * fixed mappings and modules */ #define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE) -#ifndef CONFIG_KASAN -#define VMALLOC_START (VA_START) -#else -#include -#define VMALLOC_START (KASAN_SHADOW_END + SZ_64K) -#endif - +#define VMALLOC_START (MODULES_END) #define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K) #define VMEMMAP_START (VMALLOC_END + SZ_64K) @@ -59,6 +54,7 @@ #ifndef __ASSEMBLY__ +#include #include extern void __pte_error(const char *file, int line, unsigned long val); @@ -66,8 +62,16 @@ extern void __pud_error(const char *file, int line, unsigned long val); extern void __pgd_error(const char *file, int line, unsigned long val); -#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) -#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) +#define _PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) +#define _PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) + +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +#define PROT_DEFAULT (_PROT_DEFAULT | PTE_NG) +#define PROT_SECT_DEFAULT (_PROT_SECT_DEFAULT | PMD_SECT_NG) +#else +#define PROT_DEFAULT _PROT_DEFAULT +#define PROT_SECT_DEFAULT _PROT_SECT_DEFAULT +#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ #define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE)) #define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE)) @@ -80,6 +84,7 @@ #define PROT_SECT_NORMAL_EXEC (PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL)) #define _PAGE_DEFAULT (PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL)) +#define _HYP_PAGE_DEFAULT (_PAGE_DEFAULT & ~PTE_NG) #define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE) #define PAGE_KERNEL_RO __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY) @@ -87,13 +92,13 @@ #define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE) #define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT) -#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP) +#define PAGE_HYP __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP) #define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP) #define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY) #define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN) -#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN) +#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_NG | PTE_PXN | PTE_UXN) #define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE) #define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE) #define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN) @@ -123,8 +128,8 @@ * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ -extern struct page *empty_zero_page; -#define ZERO_PAGE(vaddr) (empty_zero_page) +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page)) #define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte)) @@ -136,16 +141,6 @@ #define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) -/* Find an entry in the third-level page table. */ -#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - -#define pte_offset_kernel(dir,addr) (pmd_page_vaddr(*(dir)) + pte_index(addr)) - -#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) -#define pte_offset_map_nested(dir,addr) pte_offset_kernel((dir), (addr)) -#define pte_unmap(pte) do { } while (0) -#define pte_unmap_nested(pte) do { } while (0) - /* * The following only work if pte_present(). Undefined behaviour otherwise. */ @@ -168,6 +163,16 @@ #define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) #define pte_valid_not_user(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID) +#define pte_valid_young(pte) \ + ((pte_val(pte) & (PTE_VALID | PTE_AF)) == (PTE_VALID | PTE_AF)) + +/* + * Could the pte be present in the TLB? We must check mm_tlb_flush_pending + * so that we don't erroneously return false for pages that have been + * remapped as PROT_NONE but are yet to be flushed from the TLB. + */ +#define pte_accessible(mm, pte) \ + (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte)) static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) { @@ -218,7 +223,8 @@ static inline pte_t pte_mkcont(pte_t pte) { - return set_pte_bit(pte, __pgprot(PTE_CONT)); + pte = set_pte_bit(pte, __pgprot(PTE_CONT)); + return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE)); } static inline pte_t pte_mknoncont(pte_t pte) @@ -226,8 +232,41 @@ return clear_pte_bit(pte, __pgprot(PTE_CONT)); } +static inline pmd_t pmd_mkcont(pmd_t pmd) +{ + return __pmd(pmd_val(pmd) | PMD_SECT_CONT); +} + static inline void set_pte(pte_t *ptep, pte_t pte) { +#ifdef CONFIG_ARM64_STRICT_BREAK_BEFORE_MAKE + pteval_t old = pte_val(*ptep); + pteval_t new = pte_val(pte); + + /* Only problematic if valid -> valid */ + if (!(old & new & PTE_VALID)) + goto pte_ok; + + /* Changing attributes should go via an invalid entry */ + if (WARN_ON((old & PTE_ATTRINDX_MASK) != (new & PTE_ATTRINDX_MASK))) + goto pte_bad; + + /* Change of OA is only an issue if one mapping is writable */ + if (!(old & new & PTE_RDONLY) && + WARN_ON(pte_pfn(*ptep) != pte_pfn(pte))) + goto pte_bad; + + goto pte_ok; + +pte_bad: + *ptep = __pte(0); + dsb(ishst); + asm("tlbi vmalle1is"); + dsb(ish); + isb(); +pte_ok: +#endif + *ptep = pte; /* @@ -299,7 +338,7 @@ /* * Hugetlb definitions. */ -#define HUGE_MAX_HSTATE 2 +#define HUGE_MAX_HSTATE 4 #define HPAGE_SHIFT PMD_SHIFT #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) @@ -354,6 +393,7 @@ #define pmd_mksplitting(pmd) pte_pmd(pte_mkspecial(pmd_pte(pmd))) #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) #define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd))) +#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) #define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID)) @@ -425,13 +465,36 @@ set_pmd(pmdp, __pmd(0)); } -static inline pte_t *pmd_page_vaddr(pmd_t pmd) +static inline phys_addr_t pmd_page_paddr(pmd_t pmd) +{ + return pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK; +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) { - return __va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK); + return (unsigned long) __va(pmd_page_paddr(pmd)); } +/* Find an entry in the third-level page table. */ +#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + +#define pte_offset_phys(dir,addr) (pmd_page_paddr(*(dir)) + pte_index(addr) * sizeof(pte_t)) +#define pte_offset_kernel(dir,addr) ((pte_t *)__va(pte_offset_phys((dir), (addr)))) + +#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) +#define pte_offset_map_nested(dir,addr) pte_offset_kernel((dir), (addr)) +#define pte_unmap(pte) do { } while (0) +#define pte_unmap_nested(pte) do { } while (0) + +#define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) +#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) +#define pte_clear_fixmap() clear_fixmap(FIX_PTE) + #define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) +/* use ONLY for statically allocated translation tables */ +#define pte_offset_kimg(dir,addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr)))) + /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. @@ -458,21 +521,42 @@ set_pud(pudp, __pud(0)); } -static inline pmd_t *pud_page_vaddr(pud_t pud) +static inline phys_addr_t pud_page_paddr(pud_t pud) { - return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK); + return pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK; +} + +static inline unsigned long pud_page_vaddr(pud_t pud) +{ + return (unsigned long) __va(pud_page_paddr(pud)); } /* Find an entry in the second-level page table. */ #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) -static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) -{ - return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr); -} +#define pmd_offset_phys(dir, addr) (pud_page_paddr(*(dir)) + pmd_index(addr) * sizeof(pmd_t)) +#define pmd_offset(dir, addr) ((pmd_t *)__va(pmd_offset_phys((dir), (addr)))) + +#define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) +#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) +#define pmd_clear_fixmap() clear_fixmap(FIX_PMD) #define pud_page(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK)) +/* use ONLY for statically allocated translation tables */ +#define pmd_offset_kimg(dir,addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr)))) + +#else + +#define pud_page_paddr(pud) ({ BUILD_BUG(); 0; }) + +/* Match pmd_offset folding in */ +#define pmd_set_fixmap(addr) NULL +#define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp) +#define pmd_clear_fixmap() + +#define pmd_offset_kimg(dir,addr) ((pmd_t *)dir) + #endif /* CONFIG_PGTABLE_LEVELS > 2 */ #if CONFIG_PGTABLE_LEVELS > 3 @@ -494,21 +578,42 @@ set_pgd(pgdp, __pgd(0)); } -static inline pud_t *pgd_page_vaddr(pgd_t pgd) +static inline phys_addr_t pgd_page_paddr(pgd_t pgd) +{ + return pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK; +} + +static inline unsigned long pgd_page_vaddr(pgd_t pgd) { - return __va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK); + return (unsigned long) __va(pgd_page_paddr(pgd)); } /* Find an entry in the frst-level page table. */ #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) -static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr) -{ - return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr); -} +#define pud_offset_phys(dir, addr) (pgd_page_paddr(*(dir)) + pud_index(addr) * sizeof(pud_t)) +#define pud_offset(dir, addr) ((pud_t *)__va(pud_offset_phys((dir), (addr)))) + +#define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr)) +#define pud_set_fixmap_offset(pgd, addr) pud_set_fixmap(pud_offset_phys(pgd, addr)) +#define pud_clear_fixmap() clear_fixmap(FIX_PUD) #define pgd_page(pgd) pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK)) +/* use ONLY for statically allocated translation tables */ +#define pud_offset_kimg(dir,addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr)))) + +#else + +#define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0;}) + +/* Match pud_offset folding in */ +#define pud_set_fixmap(addr) NULL +#define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp) +#define pud_clear_fixmap() + +#define pud_offset_kimg(dir,addr) ((pud_t *)dir) + #endif /* CONFIG_PGTABLE_LEVELS > 3 */ #define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd)) @@ -516,11 +621,16 @@ /* to find an entry in a page-table-directory */ #define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) -#define pgd_offset(mm, addr) ((mm)->pgd+pgd_index(addr)) +#define pgd_offset_raw(pgd, addr) ((pgd) + pgd_index(addr)) + +#define pgd_offset(mm, addr) (pgd_offset_raw((mm)->pgd, (addr))) /* to find an entry in a kernel page-table-directory */ #define pgd_offset_k(addr) pgd_offset(&init_mm, addr) +#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) +#define pgd_clear_fixmap() clear_fixmap(FIX_PGD) + static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | @@ -649,6 +759,7 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; +extern pgd_t tramp_pg_dir[PTRS_PER_PGD]; /* * Encode and decode a swap entry: @@ -681,7 +792,8 @@ #include -#define pgtable_cache_init() do { } while (0) +void pgd_cache_init(void); +#define pgtable_cache_init pgd_cache_init /* * On AArch64, the cache coherency is handled via the set_pte_at() function. diff -ruw linux-4.4.115/arch/arm64/include/asm/pgtable-hwdef.h linux-4.4.115-fbx/arch/arm64/include/asm/pgtable-hwdef.h --- linux-4.4.115/arch/arm64/include/asm/pgtable-hwdef.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/pgtable-hwdef.h 2019-01-22 16:16:21.543228657 +0100 @@ -90,7 +90,23 @@ /* * Contiguous page definitions. */ -#define CONT_PTES (_AC(1, UL) << CONT_SHIFT) +#ifdef CONFIG_ARM64_64K_PAGES +#define CONT_PTE_SHIFT 5 +#define CONT_PMD_SHIFT 5 +#elif defined(CONFIG_ARM64_16K_PAGES) +#define CONT_PTE_SHIFT 7 +#define CONT_PMD_SHIFT 5 +#else +#define CONT_PTE_SHIFT 4 +#define CONT_PMD_SHIFT 4 +#endif + +#define CONT_PTES (1 << CONT_PTE_SHIFT) +#define CONT_PTE_SIZE (CONT_PTES * PAGE_SIZE) +#define CONT_PTE_MASK (~(CONT_PTE_SIZE - 1)) +#define CONT_PMDS (1 << CONT_PMD_SHIFT) +#define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE) +#define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1)) /* the the numerical offset of the PTE within a range of CONT_PTES */ #define CONT_RANGE_OFFSET(addr) (((addr)>>PAGE_SHIFT)&(CONT_PTES-1)) @@ -208,6 +224,8 @@ #define TCR_TG1_16K (UL(1) << 30) #define TCR_TG1_4K (UL(2) << 30) #define TCR_TG1_64K (UL(3) << 30) + +#define TCR_A1 (UL(1) << 22) #define TCR_ASID16 (UL(1) << 36) #define TCR_TBI0 (UL(1) << 37) #define TCR_HA (UL(1) << 39) diff -ruw linux-4.4.115/arch/arm64/include/asm/processor.h linux-4.4.115-fbx/arch/arm64/include/asm/processor.h --- linux-4.4.115/arch/arm64/include/asm/processor.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/processor.h 2019-01-22 16:16:21.543228657 +0100 @@ -29,8 +29,10 @@ #include +#include #include #include +#include #include #include #include @@ -49,6 +51,9 @@ #define ARCH_LOW_ADDRESS_LIMIT (arm64_dma_phys_limit - 1) #endif /* __KERNEL__ */ +extern unsigned int boot_reason; +extern unsigned int cold_boot; + struct debug_info { /* Have we suspended stepping by a debugger? */ int suspended_step; @@ -177,9 +182,11 @@ } #define ARCH_HAS_SPINLOCK_PREFETCH -static inline void spin_lock_prefetch(const void *x) +static inline void spin_lock_prefetch(const void *ptr) { - prefetchw(x); + asm volatile(ARM64_LSE_ATOMIC_INSN( + "prfm pstl1strm, %a0", + "nop") : : "p" (ptr)); } #define HAVE_ARCH_PICK_MMAP_LAYOUT @@ -187,5 +194,6 @@ #endif int cpu_enable_pan(void *__unused); +int cpu_enable_uao(void *__unused); #endif /* __ASM_PROCESSOR_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/proc-fns.h linux-4.4.115-fbx/arch/arm64/include/asm/proc-fns.h --- linux-4.4.115/arch/arm64/include/asm/proc-fns.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/proc-fns.h 2019-01-22 16:16:21.543228657 +0100 @@ -28,19 +28,17 @@ struct mm_struct; struct cpu_suspend_ctx; +extern void cpu_cache_off(void); extern void cpu_do_idle(void); extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); +extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); +void cpu_soft_restart(phys_addr_t cpu_reset, + unsigned long addr) __attribute__((noreturn)); extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr); extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr); #include -#define cpu_switch_mm(pgd,mm) \ -do { \ - BUG_ON(pgd == swapper_pg_dir); \ - cpu_do_switch_mm(virt_to_phys(pgd),mm); \ -} while (0) - #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* __ASM_PROCFNS_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/ptrace.h linux-4.4.115-fbx/arch/arm64/include/asm/ptrace.h --- linux-4.4.115/arch/arm64/include/asm/ptrace.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/ptrace.h 2019-01-22 16:16:21.543228657 +0100 @@ -121,6 +121,8 @@ u64 unused; // maintain 16 byte alignment }; +#define MAX_REG_OFFSET offsetof(struct pt_regs, pstate) + #define arch_has_single_step() (1) #ifdef CONFIG_COMPAT @@ -146,9 +148,57 @@ #define fast_interrupts_enabled(regs) \ (!((regs)->pstate & PSR_F_BIT)) -#define user_stack_pointer(regs) \ +#define GET_USP(regs) \ (!compat_user_mode(regs) ? (regs)->sp : (regs)->compat_sp) +#define SET_USP(ptregs, value) \ + (!compat_user_mode(regs) ? ((regs)->sp = value) : ((regs)->compat_sp = value)) + +extern int regs_query_register_offset(const char *name); +extern const char *regs_query_register_name(unsigned int offset); +extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n); + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset of the register. + * + * regs_get_register returns the value of a register whose offset from @regs. + * The @offset is the offset of the register in struct pt_regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline u64 regs_get_register(struct pt_regs *regs, unsigned int offset) +{ + u64 val = 0; + + offset >>= 3; + switch (offset) { + case 0 ... 30: + val = regs->regs[offset]; + break; + case offsetof(struct pt_regs, sp) >> 3: + val = regs->sp; + break; + case offsetof(struct pt_regs, pc) >> 3: + val = regs->pc; + break; + case offsetof(struct pt_regs, pstate) >> 3: + val = regs->pstate; + break; + default: + val = 0; + } + + return val; +} + +/* Valid only for Kernel mode traps. */ +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->sp; +} + static inline unsigned long regs_return_value(struct pt_regs *regs) { return regs->regs[0]; @@ -158,8 +208,15 @@ struct task_struct; int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task); -#define instruction_pointer(regs) ((unsigned long)(regs)->pc) +#define GET_IP(regs) ((unsigned long)(regs)->pc) +#define SET_IP(regs, value) ((regs)->pc = ((u64) (value))) + +#define GET_FP(ptregs) ((unsigned long)(ptregs)->regs[29]) +#define SET_FP(ptregs, value) ((ptregs)->regs[29] = ((u64) (value))) + +#include +#undef profile_pc extern unsigned long profile_pc(struct pt_regs *regs); #endif /* __ASSEMBLY__ */ diff -ruw linux-4.4.115/arch/arm64/include/asm/shmparam.h linux-4.4.115-fbx/arch/arm64/include/asm/shmparam.h --- linux-4.4.115/arch/arm64/include/asm/shmparam.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/shmparam.h 2019-01-22 16:16:21.543228657 +0100 @@ -21,7 +21,7 @@ * alignment value. Since we don't have aliasing D-caches, the rest of * the time we can safely use PAGE_SIZE. */ -#define COMPAT_SHMLBA 0x4000 +#define COMPAT_SHMLBA (4 * PAGE_SIZE) #include diff -ruw linux-4.4.115/arch/arm64/include/asm/signal32.h linux-4.4.115-fbx/arch/arm64/include/asm/signal32.h --- linux-4.4.115/arch/arm64/include/asm/signal32.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/signal32.h 2019-01-22 16:16:21.543228657 +0100 @@ -22,8 +22,6 @@ #define AARCH32_KERN_SIGRET_CODE_OFFSET 0x500 -extern const compat_ulong_t aarch32_sigret_code[6]; - int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set, struct pt_regs *regs); int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, diff -ruw linux-4.4.115/arch/arm64/include/asm/smp.h linux-4.4.115-fbx/arch/arm64/include/asm/smp.h --- linux-4.4.115/arch/arm64/include/asm/smp.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/smp.h 2019-01-22 16:16:21.543228657 +0100 @@ -16,11 +16,22 @@ #ifndef __ASM_SMP_H #define __ASM_SMP_H +#include + #include #include #include -#define raw_smp_processor_id() (current_thread_info()->cpu) +DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number); + +/* + * We don't use this_cpu_read(cpu_number) as that has implicit writes to + * preempt_count, and associated (compiler) barriers, that we'd like to avoid + * the expense of. If we're preemptible, the value can be stale at use anyway. + * And we can't use this_cpu_ptr() either, as that winds up recursing back + * here under CONFIG_DEBUG_PREEMPT=y. + */ +#define raw_smp_processor_id() (*raw_cpu_ptr(&cpu_number)) struct seq_file; @@ -57,6 +68,9 @@ */ struct secondary_data { void *stack; +#ifdef CONFIG_THREAD_INFO_IN_TASK + struct task_struct *task; +#endif }; extern struct secondary_data secondary_data; extern void secondary_entry(void); @@ -64,6 +78,15 @@ extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); +#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL +extern void arch_send_wakeup_ipi_mask(const struct cpumask *mask); +#else +static inline void arch_send_wakeup_ipi_mask(const struct cpumask *mask) +{ + BUILD_BUG(); +} +#endif + extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); diff -ruw linux-4.4.115/arch/arm64/include/asm/spinlock.h linux-4.4.115-fbx/arch/arm64/include/asm/spinlock.h --- linux-4.4.115/arch/arm64/include/asm/spinlock.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/spinlock.h 2019-10-29 09:26:22.997196856 +0100 @@ -26,9 +26,32 @@ * The memory barriers are implicit with the load-acquire and store-release * instructions. */ +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + unsigned int tmp; + arch_spinlock_t lockval; -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) + asm volatile( +" sevl\n" +"1: wfe\n" +"2: ldaxr %w0, %2\n" +" eor %w1, %w0, %w0, ror #16\n" +" cbnz %w1, 1b\n" + /* Serialise against any concurrent lockers */ + ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ +" stxr %w1, %w0, %2\n" +" nop\n" +" nop\n", + /* LSE atomics */ +" mov %w1, %w0\n" +" cas %w0, %w0, %2\n" +" eor %w1, %w1, %w0\n") +" cbnz %w1, 2b\n" + : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) + : + : "memory"); +} #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) @@ -80,7 +103,6 @@ asm volatile(ARM64_LSE_ATOMIC_INSN( /* LL/SC */ - " prfm pstl1strm, %2\n" "1: ldaxr %w0, %2\n" " eor %w1, %w0, %w0, ror #16\n" " cbnz %w1, 2f\n" diff -ruw linux-4.4.115/arch/arm64/include/asm/stacktrace.h linux-4.4.115-fbx/arch/arm64/include/asm/stacktrace.h --- linux-4.4.115/arch/arm64/include/asm/stacktrace.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/stacktrace.h 2019-01-22 16:16:21.543228657 +0100 @@ -16,14 +16,19 @@ #ifndef __ASM_STACKTRACE_H #define __ASM_STACKTRACE_H +struct task_struct; + struct stackframe { unsigned long fp; unsigned long sp; unsigned long pc; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + unsigned int graph; +#endif }; -extern int unwind_frame(struct stackframe *frame); -extern void walk_stackframe(struct stackframe *frame, +extern int unwind_frame(struct task_struct *tsk, struct stackframe *frame); +extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame, int (*fn)(struct stackframe *, void *), void *data); #endif /* __ASM_STACKTRACE_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/suspend.h linux-4.4.115-fbx/arch/arm64/include/asm/suspend.h --- linux-4.4.115/arch/arm64/include/asm/suspend.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/suspend.h 2019-01-22 16:16:21.543228657 +0100 @@ -1,7 +1,8 @@ #ifndef __ASM_SUSPEND_H #define __ASM_SUSPEND_H -#define NR_CTX_REGS 11 +#define NR_CTX_REGS 12 +#define NR_CALLEE_SAVED_REGS 12 /* * struct cpu_suspend_ctx must be 16-byte aligned since it is allocated on @@ -16,11 +17,34 @@ u64 sp; } __aligned(16); -struct sleep_save_sp { - phys_addr_t *save_ptr_stash; - phys_addr_t save_ptr_stash_phys; +/* + * Memory to save the cpu state is allocated on the stack by + * __cpu_suspend_enter()'s caller, and populated by __cpu_suspend_enter(). + * This data must survive until cpu_resume() is called. + * + * This struct desribes the size and the layout of the saved cpu state. + * The layout of the callee_saved_regs is defined by the implementation + * of __cpu_suspend_enter(), and cpu_resume(). This struct must be passed + * in by the caller as __cpu_suspend_enter()'s stack-frame is gone once it + * returns, and the data would be subsequently corrupted by the call to the + * finisher. + */ +struct sleep_stack_data { + struct cpu_suspend_ctx system_regs; + unsigned long callee_saved_regs[NR_CALLEE_SAVED_REGS]; }; +extern unsigned long *sleep_save_stash; + extern int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)); extern void cpu_resume(void); +int __cpu_suspend_enter(struct sleep_stack_data *state); +void __cpu_suspend_exit(void); +void _cpu_resume(void); + +int swsusp_arch_suspend(void); +int swsusp_arch_resume(void); +int arch_hibernation_header_save(void *addr, unsigned int max_size); +int arch_hibernation_header_restore(void *addr); + #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/sysreg.h linux-4.4.115-fbx/arch/arm64/include/asm/sysreg.h --- linux-4.4.115/arch/arm64/include/asm/sysreg.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/sysreg.h 2019-10-29 09:26:23.001196895 +0100 @@ -20,6 +20,8 @@ #ifndef __ASM_SYSREG_H #define __ASM_SYSREG_H +#include + #include /* @@ -70,20 +72,35 @@ #define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0) #define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1) +#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2) #define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0) #define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1) #define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7) #define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) +#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3) #define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\ (!!x)<<8 | 0x1f) +#define SET_PSTATE_UAO(x) __inst_arm(0xd5000000 | REG_PSTATE_UAO_IMM |\ + (!!x)<<8 | 0x1f) -/* SCTLR_EL1 */ -#define SCTLR_EL1_CP15BEN (0x1 << 5) -#define SCTLR_EL1_SED (0x1 << 8) -#define SCTLR_EL1_SPAN (0x1 << 23) +/* Common SCTLR_ELx flags. */ +#define SCTLR_ELx_EE (1 << 25) +#define SCTLR_ELx_I (1 << 12) +#define SCTLR_ELx_SA (1 << 3) +#define SCTLR_ELx_C (1 << 2) +#define SCTLR_ELx_A (1 << 1) +#define SCTLR_ELx_M 1 + +#define SCTLR_ELx_FLAGS (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \ + SCTLR_ELx_SA | SCTLR_ELx_I) + +/* SCTLR_EL1 specific flags. */ +#define SCTLR_EL1_SPAN (1 << 23) +#define SCTLR_EL1_SED (1 << 8) +#define SCTLR_EL1_CP15BEN (1 << 5) /* id_aa64isar0 */ @@ -95,6 +112,7 @@ #define ID_AA64ISAR0_AES_SHIFT 4 /* id_aa64pfr0 */ +#define ID_AA64PFR0_CSV2_SHIFT 56 #define ID_AA64PFR0_GIC_SHIFT 24 #define ID_AA64PFR0_ASIMD_SHIFT 20 #define ID_AA64PFR0_FP_SHIFT 16 @@ -135,6 +153,9 @@ #define ID_AA64MMFR1_VMIDBITS_SHIFT 4 #define ID_AA64MMFR1_HADBS_SHIFT 0 +/* id_aa64mmfr2 */ +#define ID_AA64MMFR2_UAO_SHIFT 4 + /* id_aa64dfr0 */ #define ID_AA64DFR0_CTX_CMPS_SHIFT 28 #define ID_AA64DFR0_WRPS_SHIFT 20 @@ -194,32 +215,34 @@ #ifdef __ASSEMBLY__ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 - .equ __reg_num_x\num, \num + .equ .L__reg_num_x\num, \num .endr - .equ __reg_num_xzr, 31 + .equ .L__reg_num_xzr, 31 .macro mrs_s, rt, sreg - .inst 0xd5200000|(\sreg)|(__reg_num_\rt) + .inst 0xd5200000|(\sreg)|(.L__reg_num_\rt) .endm .macro msr_s, sreg, rt - .inst 0xd5000000|(\sreg)|(__reg_num_\rt) + .inst 0xd5000000|(\sreg)|(.L__reg_num_\rt) .endm #else +#include + asm( " .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n" -" .equ __reg_num_x\\num, \\num\n" +" .equ .L__reg_num_x\\num, \\num\n" " .endr\n" -" .equ __reg_num_xzr, 31\n" +" .equ .L__reg_num_xzr, 31\n" "\n" " .macro mrs_s, rt, sreg\n" -" .inst 0xd5200000|(\\sreg)|(__reg_num_\\rt)\n" +" .inst 0xd5200000|(\\sreg)|(.L__reg_num_\\rt)\n" " .endm\n" "\n" " .macro msr_s, sreg, rt\n" -" .inst 0xd5000000|(\\sreg)|(__reg_num_\\rt)\n" +" .inst 0xd5000000|(\\sreg)|(.L__reg_num_\\rt)\n" " .endm\n" ); @@ -232,6 +255,23 @@ val |= set; asm volatile("msr sctlr_el1, %0" : : "r" (val)); } + +/* + * Unlike read_cpuid, calls to read_sysreg are never expected to be + * optimized away or replaced with synthetic values. + */ +#define read_sysreg(r) ({ \ + u64 __val; \ + asm volatile("mrs %0, " __stringify(r) : "=r" (__val)); \ + __val; \ +}) + +#define write_sysreg(v, r) do { \ + u64 __val = (u64)v; \ + asm volatile("msr " __stringify(r) ", %0" \ + : : "r" (__val)); \ +} while (0) + #endif #endif /* __ASM_SYSREG_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/system_misc.h linux-4.4.115-fbx/arch/arm64/include/asm/system_misc.h --- linux-4.4.115/arch/arm64/include/asm/system_misc.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/system_misc.h 2019-01-22 16:16:21.543228657 +0100 @@ -44,6 +44,7 @@ extern void __show_regs(struct pt_regs *); extern void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); +extern char* (*arch_read_hardware_id)(void); #define show_unhandled_signals_ratelimited() \ ({ \ diff -ruw linux-4.4.115/arch/arm64/include/asm/thread_info.h linux-4.4.115-fbx/arch/arm64/include/asm/thread_info.h --- linux-4.4.115/arch/arm64/include/asm/thread_info.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/thread_info.h 2019-10-29 09:26:23.001196895 +0100 @@ -36,22 +36,36 @@ struct task_struct; +#include #include typedef unsigned long mm_segment_t; /* * low level task data that entry.S needs immediate access to. - * __switch_to() assumes cpu_context follows immediately after cpu_domain. */ struct thread_info { unsigned long flags; /* low level flags */ mm_segment_t addr_limit; /* address limit */ +#ifndef CONFIG_THREAD_INFO_IN_TASK struct task_struct *task; /* main task structure */ +#endif +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + u64 ttbr0; /* saved TTBR0_EL1 */ +#endif int preempt_count; /* 0 => preemptable, <0 => bug */ +#ifndef CONFIG_THREAD_INFO_IN_TASK int cpu; /* cpu */ +#endif }; +#ifdef CONFIG_THREAD_INFO_IN_TASK +#define INIT_THREAD_INFO(tsk) \ +{ \ + .preempt_count = INIT_PREEMPT_COUNT, \ + .addr_limit = KERNEL_DS, \ +} +#else #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ @@ -60,25 +74,28 @@ .addr_limit = KERNEL_DS, \ } -#define init_thread_info (init_thread_union.thread_info) -#define init_stack (init_thread_union.stack) - -/* - * how to get the current stack pointer from C - */ -register unsigned long current_stack_pointer asm ("sp"); - /* * how to get the thread information struct from C */ static inline struct thread_info *current_thread_info(void) __attribute_const__; +/* + * struct thread_info can be accessed directly via sp_el0. + */ static inline struct thread_info *current_thread_info(void) { - return (struct thread_info *) - (current_stack_pointer & ~(THREAD_SIZE - 1)); + unsigned long sp_el0; + + asm ("mrs %0, sp_el0" : "=r" (sp_el0)); + + return (struct thread_info *)sp_el0; } +#define init_thread_info (init_thread_union.thread_info) +#endif + +#define init_stack (init_thread_union.stack) + #define thread_saved_pc(tsk) \ ((unsigned long)(tsk->thread.cpu_context.pc)) #define thread_saved_sp(tsk) \ @@ -113,6 +130,7 @@ #define TIF_RESTORE_SIGMASK 20 #define TIF_SINGLESTEP 21 #define TIF_32BIT 22 /* 32bit process */ +#define TIF_MM_RELEASED 24 #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) diff -ruw linux-4.4.115/arch/arm64/include/asm/tlbflush.h linux-4.4.115-fbx/arch/arm64/include/asm/tlbflush.h --- linux-4.4.115/arch/arm64/include/asm/tlbflush.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/tlbflush.h 2019-01-22 16:16:21.543228657 +0100 @@ -23,6 +23,30 @@ #include #include +#include + +/* + * Raw TLBI operations. + * + * Where necessary, use the __tlbi() macro to avoid asm() + * boilerplate. Drivers and most kernel code should use the TLB + * management routines in preference to the macro below. + * + * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending + * on whether a particular TLBI operation takes an argument or + * not. The macros handles invoking the asm with or without the + * register argument as appropriate. + */ +#define __TLBI_0(op, arg) asm ("tlbi " #op) +#define __TLBI_1(op, arg) asm ("tlbi " #op ", %0" : : "r" (arg)) +#define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg) + +#define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0) + +#define __tlbi_user(op, arg) do { \ + if (arm64_kernel_unmapped_at_el0()) \ + __tlbi(op, (arg) | USER_ASID_FLAG); \ +} while (0) /* * TLB Management @@ -66,7 +90,7 @@ static inline void local_flush_tlb_all(void) { dsb(nshst); - asm("tlbi vmalle1"); + __tlbi(vmalle1); dsb(nsh); isb(); } @@ -74,7 +98,7 @@ static inline void flush_tlb_all(void) { dsb(ishst); - asm("tlbi vmalle1is"); + __tlbi(vmalle1is); dsb(ish); isb(); } @@ -84,7 +108,8 @@ unsigned long asid = ASID(mm) << 48; dsb(ishst); - asm("tlbi aside1is, %0" : : "r" (asid)); + __tlbi(aside1is, asid); + __tlbi_user(aside1is, asid); dsb(ish); } @@ -94,7 +119,8 @@ unsigned long addr = uaddr >> 12 | (ASID(vma->vm_mm) << 48); dsb(ishst); - asm("tlbi vale1is, %0" : : "r" (addr)); + __tlbi(vale1is, addr); + __tlbi_user(vale1is, addr); dsb(ish); } @@ -121,10 +147,13 @@ dsb(ishst); for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) { - if (last_level) - asm("tlbi vale1is, %0" : : "r"(addr)); - else - asm("tlbi vae1is, %0" : : "r"(addr)); + if (last_level) { + __tlbi(vale1is, addr); + __tlbi_user(vale1is, addr); + } else { + __tlbi(vae1is, addr); + __tlbi_user(vae1is, addr); + } } dsb(ish); } @@ -149,7 +178,7 @@ dsb(ishst); for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) - asm("tlbi vaae1is, %0" : : "r"(addr)); + __tlbi(vaae1is, addr); dsb(ish); isb(); } @@ -163,7 +192,8 @@ { unsigned long addr = uaddr >> 12 | (ASID(mm) << 48); - asm("tlbi vae1is, %0" : : "r" (addr)); + __tlbi(vae1is, addr); + __tlbi_user(vae1is, addr); dsb(ish); } diff -ruw linux-4.4.115/arch/arm64/include/asm/topology.h linux-4.4.115-fbx/arch/arm64/include/asm/topology.h --- linux-4.4.115/arch/arm64/include/asm/topology.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/topology.h 2019-01-22 16:16:21.543228657 +0100 @@ -21,6 +21,16 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); +unsigned long arch_get_cpu_efficiency(int cpu); + +struct sched_domain; +#ifdef CONFIG_CPU_FREQ +#define arch_scale_freq_capacity cpufreq_scale_freq_capacity +extern unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu); +extern unsigned long cpufreq_scale_max_freq_capacity(int cpu); +#endif +#define arch_scale_cpu_capacity scale_cpu_capacity +extern unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu); #include diff -ruw linux-4.4.115/arch/arm64/include/asm/traps.h linux-4.4.115-fbx/arch/arm64/include/asm/traps.h --- linux-4.4.115/arch/arm64/include/asm/traps.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/traps.h 2019-10-29 09:26:23.001196895 +0100 @@ -34,7 +34,6 @@ void register_undef_hook(struct undef_hook *hook); void unregister_undef_hook(struct undef_hook *hook); -#ifdef CONFIG_FUNCTION_GRAPH_TRACER static inline int __in_irqentry_text(unsigned long ptr) { extern char __irqentry_text_start[]; @@ -43,12 +42,6 @@ return ptr >= (unsigned long)&__irqentry_text_start && ptr < (unsigned long)&__irqentry_text_end; } -#else -static inline int __in_irqentry_text(unsigned long ptr) -{ - return 0; -} -#endif static inline int in_exception_text(unsigned long ptr) { @@ -62,4 +55,5 @@ return in ? : __in_irqentry_text(ptr); } +static inline void get_pct_hook_init(void) {} #endif diff -ruw linux-4.4.115/arch/arm64/include/asm/uaccess.h linux-4.4.115-fbx/arch/arm64/include/asm/uaccess.h --- linux-4.4.115/arch/arm64/include/asm/uaccess.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/uaccess.h 2019-10-29 09:26:23.001196895 +0100 @@ -18,6 +18,13 @@ #ifndef __ASM_UACCESS_H #define __ASM_UACCESS_H +#include +#include +#include +#include + +#ifndef __ASSEMBLY__ + /* * User space memory access functions */ @@ -25,10 +32,8 @@ #include #include -#include #include #include -#include #include #include #include @@ -37,11 +42,11 @@ #define VERIFY_WRITE 1 /* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. + * The exception table consists of pairs of relative offsets: the first + * is the relative offset to an instruction that is allowed to fault, + * and the second is the relative offset at which the program should + * continue. No registers are modified, so it is entirely up to the + * continuation code to figure out what to do. * * All the routines below use bits of fixup code that are out of line * with the main instruction path. This means when everything is well, @@ -51,9 +56,11 @@ struct exception_table_entry { - unsigned long insn, fixup; + int insn, fixup; }; +#define ARCH_HAS_RELATIVE_EXTABLE + extern int fixup_exception(struct pt_regs *regs); #define KERNEL_DS (-1UL) @@ -65,6 +72,16 @@ static inline void set_fs(mm_segment_t fs) { current_thread_info()->addr_limit = fs; + + /* + * Enable/disable UAO so that copy_to_user() etc can access + * kernel memory with the unprivileged instructions. + */ + if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS) + asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO)); + else + asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO, + CONFIG_ARM64_UAO)); } #define segment_eq(a, b) ((a) == (b)) @@ -114,6 +131,121 @@ #define access_ok(type, addr, size) __range_ok(addr, size) #define user_addr_max get_fs +#define _ASM_EXTABLE(from, to) \ + " .pushsection __ex_table, \"a\"\n" \ + " .align 3\n" \ + " .long (" #from " - .), (" #to " - .)\n" \ + " .popsection\n" + +/* + * User access enabling/disabling. + */ +#ifdef CONFIG_ARM64_SW_TTBR0_PAN +static inline void __uaccess_ttbr0_disable(void) +{ + unsigned long flags, ttbr; + + local_irq_save(flags); + ttbr = read_sysreg(ttbr1_el1); + ttbr &= ~TTBR_ASID_MASK; + /* reserved_ttbr0 placed at the end of swapper_pg_dir */ + write_sysreg(ttbr + SWAPPER_DIR_SIZE, ttbr0_el1); + isb(); + /* Set reserved ASID */ + write_sysreg(ttbr, ttbr1_el1); + isb(); + local_irq_restore(flags); +} + +static inline void __uaccess_ttbr0_enable(void) +{ + unsigned long flags, ttbr0, ttbr1; + + /* + * Disable interrupts to avoid preemption between reading the 'ttbr0' + * variable and the MSR. A context switch could trigger an ASID + * roll-over and an update of 'ttbr0'. + */ + local_irq_save(flags); + ttbr0 = READ_ONCE(current_thread_info()->ttbr0); + + /* Restore active ASID */ + ttbr1 = read_sysreg(ttbr1_el1); + ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */ + ttbr1 |= ttbr0 & TTBR_ASID_MASK; + write_sysreg(ttbr1, ttbr1_el1); + isb(); + + /* Restore user page table */ + write_sysreg(ttbr0, ttbr0_el1); + isb(); + local_irq_restore(flags); +} + +static inline bool uaccess_ttbr0_disable(void) +{ + if (!system_uses_ttbr0_pan()) + return false; + __uaccess_ttbr0_disable(); + return true; +} + +static inline bool uaccess_ttbr0_enable(void) +{ + if (!system_uses_ttbr0_pan()) + return false; + __uaccess_ttbr0_enable(); + return true; +} +#else +static inline bool uaccess_ttbr0_disable(void) +{ + return false; +} + +static inline bool uaccess_ttbr0_enable(void) +{ + return false; +} +#endif + +#define __uaccess_disable(alt) \ +do { \ + if (!uaccess_ttbr0_disable()) \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt, \ + CONFIG_ARM64_PAN)); \ +} while (0) + +#define __uaccess_enable(alt) \ +do { \ + if (!uaccess_ttbr0_enable()) \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt, \ + CONFIG_ARM64_PAN)); \ +} while (0) + +static inline void uaccess_disable(void) +{ + __uaccess_disable(ARM64_HAS_PAN); +} + +static inline void uaccess_enable(void) +{ + __uaccess_enable(ARM64_HAS_PAN); +} + +/* + * These functions are no-ops when UAO is present. + */ +static inline void uaccess_disable_not_uao(void) +{ + __uaccess_disable(ARM64_ALT_PAN_NOT_UAO); +} + +static inline void uaccess_enable_not_uao(void) +{ + __uaccess_enable(ARM64_ALT_PAN_NOT_UAO); +} + /* * The "__xxx" versions of the user access functions do not verify the address * space - it must have been done previously with a separate "access_ok()" @@ -122,9 +254,10 @@ * The "__xxx_error" versions set the third argument to -EFAULT if an error * occurs, and leave it unchanged on success. */ -#define __get_user_asm(instr, reg, x, addr, err) \ +#define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ asm volatile( \ - "1: " instr " " reg "1, [%2]\n" \ + "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \ + alt_instr " " reg "1, [%2]\n", feature) \ "2:\n" \ " .section .fixup, \"ax\"\n" \ " .align 2\n" \ @@ -132,10 +265,7 @@ " mov %1, #0\n" \ " b 2b\n" \ " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .quad 1b, 3b\n" \ - " .previous" \ + _ASM_EXTABLE(1b, 3b) \ : "+r" (err), "=&r" (x) \ : "r" (addr), "i" (-EFAULT)) @@ -143,27 +273,29 @@ do { \ unsigned long __gu_val; \ __chk_user_ptr(ptr); \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN)); \ + uaccess_enable_not_uao(); \ switch (sizeof(*(ptr))) { \ case 1: \ - __get_user_asm("ldrb", "%w", __gu_val, (ptr), (err)); \ + __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ case 2: \ - __get_user_asm("ldrh", "%w", __gu_val, (ptr), (err)); \ + __get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ case 4: \ - __get_user_asm("ldr", "%w", __gu_val, (ptr), (err)); \ + __get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ case 8: \ - __get_user_asm("ldr", "%", __gu_val, (ptr), (err)); \ + __get_user_asm("ldr", "ldtr", "%", __gu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ default: \ BUILD_BUG(); \ } \ + uaccess_disable_not_uao(); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN)); \ } while (0) #define __get_user(x, ptr) \ @@ -190,19 +322,17 @@ ((x) = 0, -EFAULT); \ }) -#define __put_user_asm(instr, reg, x, addr, err) \ +#define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ asm volatile( \ - "1: " instr " " reg "1, [%2]\n" \ + "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \ + alt_instr " " reg "1, [%2]\n", feature) \ "2:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %w0, %3\n" \ " b 2b\n" \ " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .quad 1b, 3b\n" \ - " .previous" \ + _ASM_EXTABLE(1b, 3b) \ : "+r" (err) \ : "r" (x), "r" (addr), "i" (-EFAULT)) @@ -210,26 +340,28 @@ do { \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN)); \ + uaccess_enable_not_uao(); \ switch (sizeof(*(ptr))) { \ case 1: \ - __put_user_asm("strb", "%w", __pu_val, (ptr), (err)); \ + __put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ case 2: \ - __put_user_asm("strh", "%w", __pu_val, (ptr), (err)); \ + __put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ case 4: \ - __put_user_asm("str", "%w", __pu_val, (ptr), (err)); \ + __put_user_asm("str", "sttr", "%w", __pu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ case 8: \ - __put_user_asm("str", "%", __pu_val, (ptr), (err)); \ + __put_user_asm("str", "sttr", "%", __pu_val, (ptr), \ + (err), ARM64_HAS_UAO); \ break; \ default: \ BUILD_BUG(); \ } \ - asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN)); \ + uaccess_disable_not_uao(); \ } while (0) #define __put_user(x, ptr) \ @@ -256,24 +388,39 @@ -EFAULT; \ }) -extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n); -extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n); +extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); +extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n); extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); +static inline unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n) +{ + check_object_size(to, n, false); + return __arch_copy_from_user(to, from, n); +} + +static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n) +{ + check_object_size(from, n, true); + return __arch_copy_to_user(to, from, n); +} + static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) { - if (access_ok(VERIFY_READ, from, n)) - n = __copy_from_user(to, from, n); - else /* security hole - plug it */ + if (access_ok(VERIFY_READ, from, n)) { + check_object_size(to, n, false); + n = __arch_copy_from_user(to, from, n); + } else /* security hole - plug it */ memset(to, 0, n); return n; } static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) - n = __copy_to_user(to, from, n); + if (access_ok(VERIFY_WRITE, to, n)) { + check_object_size(from, n, true); + n = __arch_copy_to_user(to, from, n); + } return n; } @@ -299,4 +446,77 @@ extern __must_check long strlen_user(const char __user *str); extern __must_check long strnlen_user(const char __user *str, long n); +#else /* __ASSEMBLY__ */ + +#include + +/* + * User access enabling/disabling macros. + */ +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + .macro __uaccess_ttbr0_disable, tmp1 + mrs \tmp1, ttbr1_el1 // swapper_pg_dir + bic \tmp1, \tmp1, #TTBR_ASID_MASK + add \tmp1, \tmp1, #SWAPPER_DIR_SIZE // reserved_ttbr0 at the end of swapper_pg_dir + msr ttbr0_el1, \tmp1 // set reserved TTBR0_EL1 + isb + sub \tmp1, \tmp1, #SWAPPER_DIR_SIZE + msr ttbr1_el1, \tmp1 // set reserved ASID + isb + .endm + + .macro __uaccess_ttbr0_enable, tmp1, tmp2 + get_thread_info \tmp1 + ldr \tmp1, [\tmp1, #TSK_TI_TTBR0] // load saved TTBR0_EL1 + mrs \tmp2, ttbr1_el1 + extr \tmp2, \tmp2, \tmp1, #48 + ror \tmp2, \tmp2, #16 + msr ttbr1_el1, \tmp2 // set the active ASID + isb + msr ttbr0_el1, \tmp1 // set the non-PAN TTBR0_EL1 + isb + .endm + + .macro uaccess_ttbr0_disable, tmp1, tmp2 +alternative_if_not ARM64_HAS_PAN + save_and_disable_irq \tmp2 // avoid preemption + __uaccess_ttbr0_disable \tmp1 + restore_irq \tmp2 +alternative_else_nop_endif + .endm + + .macro uaccess_ttbr0_enable, tmp1, tmp2, tmp3 +alternative_if_not ARM64_HAS_PAN + save_and_disable_irq \tmp3 // avoid preemption + __uaccess_ttbr0_enable \tmp1, \tmp2 + restore_irq \tmp3 +alternative_else_nop_endif + .endm +#else + .macro uaccess_ttbr0_disable, tmp1, tmp2 + .endm + + .macro uaccess_ttbr0_enable, tmp1, tmp2, tmp3 + .endm +#endif + +/* + * These macros are no-ops when UAO is present. + */ + .macro uaccess_disable_not_uao, tmp1, tmp2 + uaccess_ttbr0_disable \tmp1, \tmp2 +alternative_if ARM64_ALT_PAN_NOT_UAO + SET_PSTATE_PAN(1) +alternative_else_nop_endif + .endm + + .macro uaccess_enable_not_uao, tmp1, tmp2, tmp3 + uaccess_ttbr0_enable \tmp1, \tmp2, \tmp3 +alternative_if ARM64_ALT_PAN_NOT_UAO + SET_PSTATE_PAN(0) +alternative_else_nop_endif + .endm + +#endif /* __ASSEMBLY__ */ + #endif /* __ASM_UACCESS_H */ diff -ruw linux-4.4.115/arch/arm64/include/asm/vdso_datapage.h linux-4.4.115-fbx/arch/arm64/include/asm/vdso_datapage.h --- linux-4.4.115/arch/arm64/include/asm/vdso_datapage.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/vdso_datapage.h 2019-01-22 16:16:21.543228657 +0100 @@ -22,6 +22,8 @@ struct vdso_data { __u64 cs_cycle_last; /* Timebase at clocksource init */ + __u64 raw_time_sec; /* Raw time */ + __u64 raw_time_nsec; __u64 xtime_clock_sec; /* Kernel time */ __u64 xtime_clock_nsec; __u64 xtime_coarse_sec; /* Coarse time */ @@ -29,8 +31,10 @@ __u64 wtm_clock_sec; /* Wall to monotonic time */ __u64 wtm_clock_nsec; __u32 tb_seq_count; /* Timebase sequence counter */ - __u32 cs_mult; /* Clocksource multiplier */ - __u32 cs_shift; /* Clocksource shift */ + /* cs_* members must be adjacent and in this order (ldp accesses) */ + __u32 cs_mono_mult; /* NTP-adjusted clocksource multiplier */ + __u32 cs_shift; /* Clocksource shift (mono = raw) */ + __u32 cs_raw_mult; /* Raw clocksource multiplier */ __u32 tz_minuteswest; /* Whacky timezone stuff */ __u32 tz_dsttime; __u32 use_syscall; diff -ruw linux-4.4.115/arch/arm64/include/asm/virt.h linux-4.4.115-fbx/arch/arm64/include/asm/virt.h --- linux-4.4.115/arch/arm64/include/asm/virt.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/virt.h 2019-10-29 09:26:23.001196895 +0100 @@ -18,11 +18,29 @@ #ifndef __ASM__VIRT_H #define __ASM__VIRT_H +/* + * The arm64 hcall implementation uses x0 to specify the hcall type. A value + * less than 0xfff indicates a special hcall, such as get/set vector. + * Any other value is used as a pointer to the function to call. + */ + +/* HVC_GET_VECTORS - Return the value of the vbar_el2 register. */ +#define HVC_GET_VECTORS 0 + +/* + * HVC_SET_VECTORS - Set the value of the vbar_el2 register. + * + * @x1: Physical address of the new vector table. + */ +#define HVC_SET_VECTORS 1 + #define BOOT_CPU_MODE_EL1 (0xe11) #define BOOT_CPU_MODE_EL2 (0xe12) #ifndef __ASSEMBLY__ +#include + /* * __boot_cpu_mode records what mode CPUs were booted in. * A correctly-implemented bootloader must start all CPUs in the same mode: @@ -50,6 +68,14 @@ return __boot_cpu_mode[0] != __boot_cpu_mode[1]; } +static inline bool is_kernel_in_hyp_mode(void) +{ + u64 el; + + asm("mrs %0, CurrentEL" : "=r" (el)); + return el == CurrentEL_EL2; +} + /* The section containing the hypervisor text */ extern char __hyp_text_start[]; extern char __hyp_text_end[]; diff -ruw linux-4.4.115/arch/arm64/include/asm/word-at-a-time.h linux-4.4.115-fbx/arch/arm64/include/asm/word-at-a-time.h --- linux-4.4.115/arch/arm64/include/asm/word-at-a-time.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/asm/word-at-a-time.h 2019-01-22 16:16:21.543228657 +0100 @@ -16,6 +16,8 @@ #ifndef __ASM_WORD_AT_A_TIME_H #define __ASM_WORD_AT_A_TIME_H +#include + #ifndef __AARCH64EB__ #include @@ -81,10 +83,7 @@ #endif " b 2b\n" " .popsection\n" - " .pushsection __ex_table,\"a\"\n" - " .align 3\n" - " .quad 1b, 3b\n" - " .popsection" + _ASM_EXTABLE(1b, 3b) : "=&r" (ret), "=&r" (offset) : "r" (addr), "Q" (*(unsigned long *)addr)); diff -ruw linux-4.4.115/arch/arm64/include/uapi/asm/ptrace.h linux-4.4.115-fbx/arch/arm64/include/uapi/asm/ptrace.h --- linux-4.4.115/arch/arm64/include/uapi/asm/ptrace.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/include/uapi/asm/ptrace.h 2019-01-22 16:16:21.547228694 +0100 @@ -45,6 +45,7 @@ #define PSR_A_BIT 0x00000100 #define PSR_D_BIT 0x00000200 #define PSR_PAN_BIT 0x00400000 +#define PSR_UAO_BIT 0x00800000 #define PSR_Q_BIT 0x08000000 #define PSR_V_BIT 0x10000000 #define PSR_C_BIT 0x20000000 diff -ruw linux-4.4.115/arch/arm64/Kconfig linux-4.4.115-fbx/arch/arm64/Kconfig --- linux-4.4.115/arch/arm64/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/Kconfig 2019-10-29 09:32:37.124855595 +0100 @@ -13,6 +13,8 @@ select ARCH_WANT_OPTIONAL_GPIOLIB select ARCH_WANT_COMPAT_IPC_PARSE_VERSION select ARCH_WANT_FRAME_POINTERS + select ARCH_HAVE_CUSTOM_GPIO_H + select ARCH_HAS_UBSAN_SANITIZE_ALL select ARM_AMBA select ARM_ARCH_TIMER select ARM_GIC @@ -23,12 +25,13 @@ select ARM_PSCI_FW select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS - select COMMON_CLK + select COMMON_CLK if !ARCH_QCOM select CPU_PM if (SUSPEND || CPU_IDLE) select DCACHE_WORD_ACCESS select EDAC_SUPPORT select FRAME_POINTER select GENERIC_ALLOCATOR + select EDAC_SUPPORT select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS_BROADCAST select GENERIC_CPU_AUTOPROBE @@ -48,9 +51,13 @@ select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_BITREVERSE + select HAVE_ARCH_HARDENED_USERCOPY + select HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP && !(ARM64_16K_PAGES && ARM64_VA_BITS_48) select HAVE_ARCH_KGDB + select HAVE_ARCH_MMAP_RND_BITS + select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_BPF_JIT @@ -70,15 +77,18 @@ select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_GENERIC_DMA_COHERENT - select HAVE_HW_BREAKPOINT if PERF_EVENTS + select HAVE_IRQ_TIME_ACCOUNTING select HAVE_MEMBLOCK select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RCU_TABLE_FREE select HAVE_SYSCALL_TRACEPOINTS - select IOMMU_DMA if IOMMU_SUPPORT + select IOMMU_DMA if (IOMMU_SUPPORT && !ARCH_QCOM) + select HAVE_KPROBES + select HAVE_KRETPROBES if HAVE_KPROBES select IRQ_DOMAIN select IRQ_FORCED_THREADING select MODULES_USE_ELF_RELA @@ -93,6 +103,8 @@ select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE select HAVE_CONTEXT_TRACKING + select HAVE_ARM_SMCCC + select THREAD_INFO_IN_TASK help ARM 64-bit (AArch64) Linux support. @@ -105,9 +117,40 @@ config MMU def_bool y +config ARCH_MMAP_RND_BITS_MIN + default 14 if ARM64_64K_PAGES + default 16 if ARM64_16K_PAGES + default 18 + +# max bits determined by the following formula: +# VA_BITS - PAGE_SHIFT - 3 +config ARCH_MMAP_RND_BITS_MAX + default 19 if ARM64_VA_BITS=36 + default 24 if ARM64_VA_BITS=39 + default 27 if ARM64_VA_BITS=42 + default 30 if ARM64_VA_BITS=47 + default 29 if ARM64_VA_BITS=48 && ARM64_64K_PAGES + default 31 if ARM64_VA_BITS=48 && ARM64_16K_PAGES + default 33 if ARM64_VA_BITS=48 + default 14 if ARM64_64K_PAGES + default 16 if ARM64_16K_PAGES + default 18 + +config ARCH_MMAP_RND_COMPAT_BITS_MIN + default 7 if ARM64_64K_PAGES + default 9 if ARM64_16K_PAGES + default 11 + +config ARCH_MMAP_RND_COMPAT_BITS_MAX + default 16 + config NO_IOPORT_MAP def_bool y if !PCI +config ILLEGAL_POINTER_VALUE + hex + default 0xdead000000000000 + config STACKTRACE_SUPPORT def_bool y @@ -159,6 +202,32 @@ config SMP def_bool y +config ARM64_DMA_USE_IOMMU + bool + select ARM_HAS_SG_CHAIN + select NEED_SG_DMA_LENGTH + +if ARM64_DMA_USE_IOMMU + +config ARM64_DMA_IOMMU_ALIGNMENT + int "Maximum PAGE_SIZE order of alignment for DMA IOMMU buffers" + range 4 9 + default 9 + help + DMA mapping framework by default aligns all buffers to the smallest + PAGE_SIZE order which is greater than or equal to the requested buffer + size. This works well for buffers up to a few hundreds kilobytes, but + for larger buffers it just a waste of address space. Drivers which has + relatively small addressing window (like 64Mib) might run out of + virtual space with just a few allocations. + + With this parameter you can specify the maximum PAGE_SIZE order for + DMA IOMMU buffers. Larger buffers will be aligned only to this + specified order. The order is expressed as a power of two multiplied + by the PAGE_SIZE. + +endif + config SWIOTLB def_bool y @@ -180,6 +249,15 @@ default 3 if ARM64_16K_PAGES && ARM64_VA_BITS_47 default 4 if !ARM64_64K_PAGES && ARM64_VA_BITS_48 +config MSM_GVM_QUIN + bool "Enable virtualization Support for MSM kernel required for QUIN platform" + help + This enables support for MSM Kernel based virtual + machine for QUIN platform. + This helps to enable virtual driver support. + This should work on 64bit machine. + If you don't know what to do here, say N. + source "init/Kconfig" source "kernel/Kconfig.freezer" @@ -363,6 +441,7 @@ bool "Cortex-A53: 843419: A load or store might access an incorrect address" depends on MODULES default y + select ARM64_MODULE_CMODEL_LARGE help This option builds kernel modules using the large memory model in order to avoid the use of the ADRP instruction, which can cause @@ -443,6 +522,25 @@ requires applications compiled with 16K (or a multiple of 16K) aligned segments. +config ARM64_DCACHE_DISABLE + bool "Disable CPU Data Caches" + help + Disable CPU data cache usage by setting the SCTLR[C] bit during + kernel initialization. This will result in a considerable + performance impact, but may be useful in certain situations. + + If you are not sure what to do, select 'N' here. + +config ARM64_ICACHE_DISABLE + bool "Disable CPU Instruction Caches" + help + Disable CPU instruction cache usage by setting the SCTLR[I] + bit during kernel initialization. This will result in a + considerable performance impact, but may be useful in certain + situations. + + If you are not sure what to do, select 'N' here. + config ARM64_64K_PAGES bool "64KB" help @@ -453,6 +551,37 @@ endchoice +config MSM_APP_API + bool "API support to enable / disable app settings for MSM8996" + depends on ARCH_MSM8996 && (ENABLE_FP_SIMD_SETTINGS || MSM_APP_SETTINGS) + help + Add API support to enable / disable the app settings to be used + at runtime. These APIs are used to enable / disable app setting + when specific aarch32 or aarch64 processes are running. + + If you are not sure what to do, select 'N' here. + +config ENABLE_FP_SIMD_SETTINGS + bool "Enable FP(Floating Point) Settings for Qualcomm MSM8996" + depends on ARCH_MSM8996 + select MSM_APP_API + help + Enable FP(Floating Point) and SIMD settings for the MSM8996 during + the execution of the aarch32 processes and disable these settings + when you switch to the aarch64 processes. + + If you are not sure what to do, select 'N' here. + +config MSM_APP_SETTINGS + bool "Support to enable / disable app settings for MSM8996" + depends on ARCH_MSM8996 + select MSM_APP_API + help + Expose an interface used by the userspace at runtime to + enable / disable the app specific settings. + + If you are not sure what to do, select 'N' here. + choice prompt "Virtual address space size" default ARM64_VA_BITS_39 if ARM64_4K_PAGES @@ -524,9 +653,38 @@ Say Y here to experiment with turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu. +config ARCH_ENABLE_MEMORY_HOTPLUG + depends on !NUMA + def_bool y + +config ARCH_ENABLE_MEMORY_HOTREMOVE + def_bool y + +# The GPIO number here must be sorted by descending number. In case of +# a multiplatform kernel, we just want the highest value required by the +# selected platforms. +config ARCH_NR_GPIO + int + default 1024 if ARCH_TEGRA + default 1024 if ARCH_QCOM + default 256 + help + Maximum number of GPIOs in the system. + + If unsure, leave the default value. + +config QCOM_TLB_EL2_HANDLER + bool "Raise TLB conflict exception to EL2" + help + This option enables TLB conflict to be handled + by EL2. + source kernel/Kconfig.preempt source kernel/Kconfig.hz +config ARCH_SUPPORTS_DEBUG_PAGEALLOC + def_bool y + config ARCH_HAS_HOLES_MEMORYMODEL def_bool y if SPARSEMEM @@ -547,10 +705,34 @@ def_bool y depends on ARM_PMU -config SYS_SUPPORTS_HUGETLBFS - def_bool y +config ARM64_REG_REBALANCE_ON_CTX_SW + bool "Rebalance registers during context switches." + def_bool ARCH_MSM8996 + help + Forcefully re-balance register rename pools on context switches for + improved performance on some devices. + +config PERF_EVENTS_USERMODE + bool "Enable usermode access for perf events" + depends on PERF_EVENTS + help + Enable user-mode access to performance counters for perf events. + If enabled, the access permissions allowing CPU performance + counters to be accessed from user-mode are set. + + If you want user-mode programs to access perf events, say Y + +config PERF_EVENTS_RESET_PMU_DEBUGFS + bool "Reset PMU via debugfs node" + depends on PERF_EVENTS + help + Enable the debugfs node that can be used to reset PMUs and all + state variables associated with PMUs. If enabled, PMU and internal + state variable are cleared. + If you want to reset PMU and PMU related internal Perf variables + via debugfs then say Y. -config ARCH_WANT_GENERAL_HUGETLB +config SYS_SUPPORTS_HUGETLBFS def_bool y config ARCH_WANT_HUGE_PMD_SHARE @@ -564,6 +746,10 @@ source "mm/Kconfig" +config ARCH_MEMORY_PROBE + def_bool y + depends on MEMORY_HOTPLUG + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" ---help--- @@ -611,6 +797,35 @@ However for 4K, we choose a higher default value, 11 as opposed to 10, giving us 4M allocations matching the default size used by generic code. +config UNMAP_KERNEL_AT_EL0 + bool "Unmap kernel when running in userspace (aka \"KAISER\")" if EXPERT + default y + help + Speculation attacks against some high-performance processors can + be used to bypass MMU permission checks and leak kernel data to + userspace. This can be defended against by unmapping the kernel + when running in userspace, mapping it back in on exception entry + via a trampoline page in the vector table. + + If unsure, say Y. + +config HARDEN_BRANCH_PREDICTOR + bool "Harden the branch predictor against aliasing attacks" if EXPERT + default y + help + Speculation attacks against some high-performance processors rely on + being able to manipulate the branch predictor for a victim context by + executing aliasing branches in the attacker context. Such attacks + can be partially mitigated against by clearing internal branch + predictor state and limiting the prediction logic in some situations. + + This config option will take CPU-specific actions to harden the + branch predictor against aliasing attacks and may rely on specific + instruction sequences or control bits being set by the system + firmware. + + If unsure, say Y. + menuconfig ARMV8_DEPRECATED bool "Emulate deprecated/obsolete ARMv8 instructions" depends on COMPAT @@ -678,11 +893,18 @@ If unsure, say Y endif +config ARM64_SW_TTBR0_PAN + bool "Emulate Privileged Access Never using TTBR0_EL1 switching" + help + Enabling this option prevents the kernel from accessing + user-space memory directly by pointing TTBR0_EL1 to a reserved + zeroed area and reserved ASID. The user access routines + restore the valid TTBR0_EL1 temporarily. + menu "ARMv8.1 architectural features" config ARM64_HW_AFDBM bool "Support for hardware updates of the Access and Dirty page flags" - default y help The ARMv8.1 architecture extensions introduce support for hardware updates of the access and dirty information in page @@ -699,7 +921,6 @@ config ARM64_PAN bool "Enable support for Privileged Access Never (PAN)" - default y help Privileged Access Never (PAN; part of the ARMv8.1 Extensions) prevents the kernel or hypervisor from accessing user-space (EL0) @@ -725,10 +946,92 @@ endmenu +config ARM64_UAO + bool "Enable support for User Access Override (UAO)" + help + User Access Override (UAO; part of the ARMv8.2 Extensions) + causes the 'unprivileged' variant of the load/store instructions to + be overriden to be privileged. + + This option changes get_user() and friends to use the 'unprivileged' + variant of the load/store instructions. This ensures that user-space + really did have access to the supplied memory. When addr_limit is + set to kernel memory the UAO bit will be set, allowing privileged + access to kernel memory. + + Choosing this option will cause copy_to_user() et al to use user-space + memory permissions. + + The feature is detected at runtime, the kernel will use the + regular load/store instructions if the cpu does not implement the + feature. + +config ARM64_MODULE_CMODEL_LARGE + bool + +config ARM64_MODULE_PLTS + bool + select ARM64_MODULE_CMODEL_LARGE + select HAVE_MOD_ARCH_SPECIFIC + +config RELOCATABLE + bool + help + This builds the kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since AArch64 uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + +config RANDOMIZE_BASE + bool "Randomize the address of the kernel image" + select ARM64_MODULE_PLTS if MODULES + select RELOCATABLE + help + Randomizes the virtual address at which the kernel image is + loaded, as a security feature that deters exploit attempts + relying on knowledge of the location of kernel internals. + + It is the bootloader's job to provide entropy, by passing a + random u64 value in /chosen/kaslr-seed at kernel entry. + + When booting via the UEFI stub, it will invoke the firmware's + EFI_RNG_PROTOCOL implementation (if available) to supply entropy + to the kernel proper. In addition, it will randomise the physical + location of the kernel Image as well. + + If unsure, say N. + +config RANDOMIZE_MODULE_REGION_FULL + bool "Randomize the module region independently from the core kernel" + depends on RANDOMIZE_BASE && !DYNAMIC_FTRACE + default y + help + Randomizes the location of the module region without considering the + location of the core kernel. This way, it is impossible for modules + to leak information about the location of core kernel data structures + but it does imply that function calls between modules and the core + kernel will need to be resolved via veneers in the module PLT. + + When this option is not set, the module region will be randomized over + a limited range that contains the [_stext, _etext] interval of the + core kernel, so branch relocations are always in range. + endmenu menu "Boot options" +config ARM64_ACPI_PARKING_PROTOCOL + bool "Enable support for the ARM64 ACPI parking protocol" + depends on ACPI + help + Enable support for the ARM64 ACPI parking protocol. If disabled + the kernel will not allow booting through the ARM64 ACPI parking + protocol even if the corresponding data is present in the ACPI + MADT table. + config CMDLINE string "Default kernel command string" default "" @@ -737,6 +1040,23 @@ entering them here. As a minimum, you should specify the the root device (e.g. root=/dev/nfs). +choice + prompt "Kernel command line type" if CMDLINE != "" + default CMDLINE_FROM_BOOTLOADER + +config CMDLINE_FROM_BOOTLOADER + bool "Use bootloader kernel arguments if available" + help + Uses the command-line options passed by the boot loader. If + the boot loader doesn't provide any, the default kernel command + string provided in CMDLINE will be used. + +config CMDLINE_EXTEND + bool "Extend bootloader kernel arguments" + help + The command-line arguments provided by the boot loader will be + appended to the default kernel command string. + config CMDLINE_FORCE bool "Always use the default kernel command string" help @@ -744,6 +1064,7 @@ loader passes other arguments to the kernel. This is useful if you cannot or don't want to change the command-line options your boot loader passes to the kernel. +endchoice config EFI_STUB bool @@ -776,6 +1097,46 @@ However, even with this option, the resultant kernel should continue to boot on existing non-UEFI platforms. +config BUILD_ARM64_APPENDED_DTB_IMAGE + bool "Build a concatenated Image.gz/dtb by default" + depends on OF + help + Enabling this option will cause a concatenated Image.gz and list of + DTBs to be built by default (instead of a standalone Image.gz.) + The image will built in arch/arm64/boot/Image.gz-dtb + +choice + prompt "Appended DTB Kernel Image name" + depends on BUILD_ARM64_APPENDED_DTB_IMAGE + help + Enabling this option will cause a specific kernel image Image or + Image.gz to be used for final image creation. + The image will built in arch/arm64/boot/IMAGE-NAME-dtb + + config IMG_GZ_DTB + bool "Image.gz-dtb" + config IMG_DTB + bool "Image-dtb" +endchoice + +config BUILD_ARM64_APPENDED_KERNEL_IMAGE_NAME + string + depends on BUILD_ARM64_APPENDED_DTB_IMAGE + default "Image.gz-dtb" if IMG_GZ_DTB + default "Image-dtb" if IMG_DTB + +config BUILD_ARM64_APPENDED_DTB_IMAGE_NAMES + string "Default dtb names" + depends on BUILD_ARM64_APPENDED_DTB_IMAGE + help + Space separated list of names of dtbs to append when + building a concatenated Image.gz-dtb. + +config BUILD_ARM64_DT_OVERLAY + bool "enable DT overlay compilation support" + depends on OF + help + This option enables support for DT overlay compilation. endmenu menu "Userspace binary formats" @@ -811,6 +1172,14 @@ source "kernel/power/Kconfig" +config ARCH_HIBERNATION_POSSIBLE + def_bool y + depends on CPU_PM + +config ARCH_HIBERNATION_HEADER + def_bool y + depends on HIBERNATION + config ARCH_SUSPEND_POSSIBLE def_bool y diff -ruw linux-4.4.115/arch/arm64/Kconfig.debug linux-4.4.115-fbx/arch/arm64/Kconfig.debug --- linux-4.4.115/arch/arm64/Kconfig.debug 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/Kconfig.debug 2019-01-22 16:16:21.523228476 +0100 @@ -64,16 +64,16 @@ config DEBUG_RODATA bool "Make kernel text and rodata read-only" + default y help If this is set, kernel text and rodata will be made read-only. This is to help catch accidental or malicious attempts to change the - kernel's executable code. Additionally splits rodata from kernel - text so it can be made explicitly non-executable. + kernel's executable code. If in doubt, say Y config DEBUG_ALIGN_RODATA - depends on DEBUG_RODATA && ARM64_4K_PAGES + depends on DEBUG_RODATA bool "Align linker sections up to SECTION_SIZE" help If this option is enabled, sections that may potentially be marked as @@ -85,6 +85,46 @@ If in doubt, say N +config FORCE_PAGES + bool "Force lowmem to be mapped with 4K pages" + help + There are some advanced debug features that can only be done when + memory is mapped with pages instead of sections. Enable this option + to always map lowmem pages with pages. This may have a performance + cost due to increased TLB pressure. + + If unsure say N. + +config FREE_PAGES_RDONLY + bool "Set pages as read only while on the buddy list" + select FORCE_PAGES + select DEBUG_PAGEALLOC + help + Pages are always mapped in the kernel. This means that anyone + can write to the page if they have the address. Enable this option + to mark pages as read only to trigger a fault if any code attempts + to write to a page on the buddy list. This may have a performance + impact. + + If unsure, say N. + +config KERNEL_TEXT_RDONLY + bool "Set kernel text section pages as read only" + depends on FREE_PAGES_RDONLY + depends on !DEBUG_RODATA + help + The kernel text pages are always mapped in the kernel. + This means that anyone can write to the page if they have + the address. Enable this option to mark the kernel text pages + as read only to trigger a fault if any code attempts to write + to a page part of the kernel text section. This may have a + performance impact. + + If unsure, say N. + +config ARM64_STRICT_BREAK_BEFORE_MAKE + bool "Enforce strict break-before-make on page table updates " + source "drivers/hwtracing/coresight/Kconfig" endmenu diff -ruw linux-4.4.115/arch/arm64/Kconfig.platforms linux-4.4.115-fbx/arch/arm64/Kconfig.platforms --- linux-4.4.115/arch/arm64/Kconfig.platforms 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/Kconfig.platforms 2019-10-29 09:26:22.989196778 +0100 @@ -49,9 +49,65 @@ config ARCH_QCOM bool "Qualcomm Platforms" select PINCTRL + select CLKDEV_LOOKUP + select HAVE_CLK + select HAVE_CLK_PREPARE + select PM_OPP + select SOC_BUS + select MSM_IRQ + select THERMAL_WRITABLE_TRIPS + select RATIONAL + select ARCH_HAS_RESET_CONTROLLER help This enables support for the ARMv8 based Qualcomm chipsets. +config ARCH_MSM8996 + bool "Enable Support for Qualcomm MSM8996" + depends on ARCH_QCOM + select COMMON_CLK_MSM + help + This enables support for the MSM8996 chipset. If you do not + wish to build a kernel that runs on this chipset, say 'N' here. + +config ARCH_MSM8998 + bool "Enable Support for Qualcomm MSM8998" + depends on ARCH_QCOM + select COMMON_CLK_MSM + help + This enables support for the MSM8998 chipset. If you do not + wish to build a kernel that runs on this chipset, say 'N' here. + +config ARCH_MSMHAMSTER + bool "Enable Support for Qualcomm Technologies Inc MSMHAMSTER" + depends on ARCH_QCOM + select COMMON_CLK_MSM + help + This enables support for the MSMHAMSTER chipset. + If you do not wish to build a kernel that runs + on this chipset,say 'N' here. + +config ARCH_SDM660 + bool "Enable Support for Qualcomm Technologies Inc SDM660" + depends on ARCH_QCOM + select COMMON_CLK + select COMMON_CLK_QCOM + select QCOM_GDSC + help + This enables support for the SDM660 chipset. + If you do not wish to build a kernel that runs + on this chipset,say 'N' here. + +config ARCH_SDM630 + bool "Enable Support for Qualcomm Technologies Inc SDM630" + depends on ARCH_QCOM + select COMMON_CLK + select COMMON_CLK_QCOM + select QCOM_GDSC + help + This enables support for the SDM630 chipset. + If you do not wish to build a kernel that runs + on this chipset,say 'N' here. + config ARCH_ROCKCHIP bool "Rockchip Platforms" select ARCH_HAS_RESET_CONTROLLER diff -ruw linux-4.4.115/arch/arm64/kernel/alternative.c linux-4.4.115-fbx/arch/arm64/kernel/alternative.c --- linux-4.4.115/arch/arm64/kernel/alternative.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/alternative.c 2019-10-29 09:26:23.001196895 +0100 @@ -108,7 +108,7 @@ for (i = 0; i < nr_inst; i++) { insn = get_alt_insn(alt, origptr + i, replptr + i); - *(origptr + i) = cpu_to_le32(insn); + BUG_ON(aarch64_insn_patch_text_nosync(origptr + i, insn)); } flush_icache_range((uintptr_t)origptr, @@ -158,9 +158,3 @@ __apply_alternatives(®ion); } - -void free_alternatives_memory(void) -{ - free_reserved_area(__alt_instructions, __alt_instructions_end, - 0, "alternatives"); -} diff -ruw linux-4.4.115/arch/arm64/kernel/arm64ksyms.c linux-4.4.115-fbx/arch/arm64/kernel/arm64ksyms.c --- linux-4.4.115/arch/arm64/kernel/arm64ksyms.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/arm64ksyms.c 2019-01-22 16:16:21.547228694 +0100 @@ -26,15 +26,18 @@ #include #include #include +#include +#include +#include #include EXPORT_SYMBOL(copy_page); EXPORT_SYMBOL(clear_page); /* user mem (segment) */ -EXPORT_SYMBOL(__copy_from_user); -EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(__arch_copy_from_user); +EXPORT_SYMBOL(__arch_copy_to_user); EXPORT_SYMBOL(__clear_user); EXPORT_SYMBOL(__copy_in_user); @@ -67,4 +70,13 @@ #ifdef CONFIG_FUNCTION_TRACER EXPORT_SYMBOL(_mcount); +NOKPROBE_SYMBOL(_mcount); #endif + /* caching functions */ +EXPORT_SYMBOL(__dma_inv_range); +EXPORT_SYMBOL(__dma_clean_range); +EXPORT_SYMBOL(__dma_flush_range); + + /* arm-smccc */ +EXPORT_SYMBOL(arm_smccc_smc); +EXPORT_SYMBOL(arm_smccc_hvc); diff -ruw linux-4.4.115/arch/arm64/kernel/armv8_deprecated.c linux-4.4.115-fbx/arch/arm64/kernel/armv8_deprecated.c --- linux-4.4.115/arch/arm64/kernel/armv8_deprecated.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/armv8_deprecated.c 2019-01-22 16:16:21.547228694 +0100 @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -62,7 +61,7 @@ }; static LIST_HEAD(insn_emulation); -static int nr_insn_emulated; +static int nr_insn_emulated __initdata; static DEFINE_RAW_SPINLOCK(insn_emulation_lock); static void register_emulation_hooks(struct insn_emulation_ops *ops) @@ -173,7 +172,7 @@ return ret; } -static void register_insn_emulation(struct insn_emulation_ops *ops) +static void __init register_insn_emulation(struct insn_emulation_ops *ops) { unsigned long flags; struct insn_emulation *insn; @@ -237,7 +236,7 @@ { } }; -static void register_insn_emulation_sysctl(struct ctl_table *table) +static void __init register_insn_emulation_sysctl(struct ctl_table *table) { unsigned long flags; int i = 0; @@ -281,9 +280,9 @@ * Error-checking SWP macros implemented using ldxr{b}/stxr{b} */ #define __user_swpX_asm(data, addr, res, temp, B) \ +do { \ + uaccess_enable(); \ __asm__ __volatile__( \ - ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ "0: ldxr"B" %w2, [%3]\n" \ "1: stxr"B" %w0, %w1, [%3]\n" \ " cbz %w0, 2f\n" \ @@ -297,17 +296,14 @@ "4: mov %w0, %w5\n" \ " b 3b\n" \ " .popsection" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .quad 0b, 4b\n" \ - " .quad 1b, 4b\n" \ - " .popsection\n" \ - ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) \ + _ASM_EXTABLE(0b, 4b) \ + _ASM_EXTABLE(1b, 4b) \ : "=&r" (res), "+r" (data), "=&r" (temp) \ : "r" ((unsigned long)addr), "i" (-EAGAIN), \ "i" (-EFAULT) \ - : "memory") + : "memory"); \ + uaccess_disable(); \ +} while (0) #define __user_swp_asm(data, addr, res, temp) \ __user_swpX_asm(data, addr, res, temp, "") @@ -370,6 +366,21 @@ return res; } +#define ARM_OPCODE_CONDITION_UNCOND 0xf + +static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr) +{ + u32 cc_bits = opcode >> 28; + + if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) { + if ((*aarch32_opcode_cond_checks[cc_bits])(psr)) + return ARM_OPCODE_CONDTEST_PASS; + else + return ARM_OPCODE_CONDTEST_FAIL; + } + return ARM_OPCODE_CONDTEST_UNCOND; +} + /* * swp_handler logs the id of calling process, dissects the instruction, sanity * checks the memory location, calls emulate_swpX for the actual operation and @@ -384,7 +395,7 @@ type = instr & TYPE_SWPB; - switch (arm_check_condition(instr, regs->pstate)) { + switch (aarch32_check_condition(instr, regs->pstate)) { case ARM_OPCODE_CONDTEST_PASS: break; case ARM_OPCODE_CONDTEST_FAIL: @@ -465,7 +476,7 @@ { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); - switch (arm_check_condition(instr, regs->pstate)) { + switch (aarch32_check_condition(instr, regs->pstate)) { case ARM_OPCODE_CONDTEST_PASS: break; case ARM_OPCODE_CONDTEST_FAIL: diff -ruw linux-4.4.115/arch/arm64/kernel/asm-offsets.c linux-4.4.115-fbx/arch/arm64/kernel/asm-offsets.c --- linux-4.4.115/arch/arm64/kernel/asm-offsets.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/asm-offsets.c 2019-01-22 16:16:21.547228694 +0100 @@ -22,22 +22,33 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include int main(void) { DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); BLANK(); +#ifdef CONFIG_THREAD_INFO_IN_TASK + DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); + DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); + DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit)); + DEFINE(TSK_STACK, offsetof(struct task_struct, stack)); +#else DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); - DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); +#endif +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + DEFINE(TSK_TI_TTBR0, offsetof(struct thread_info, ttbr0)); +#endif BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); BLANK(); @@ -49,6 +60,17 @@ DEFINE(S_X5, offsetof(struct pt_regs, regs[5])); DEFINE(S_X6, offsetof(struct pt_regs, regs[6])); DEFINE(S_X7, offsetof(struct pt_regs, regs[7])); + DEFINE(S_X8, offsetof(struct pt_regs, regs[8])); + DEFINE(S_X10, offsetof(struct pt_regs, regs[10])); + DEFINE(S_X12, offsetof(struct pt_regs, regs[12])); + DEFINE(S_X14, offsetof(struct pt_regs, regs[14])); + DEFINE(S_X16, offsetof(struct pt_regs, regs[16])); + DEFINE(S_X18, offsetof(struct pt_regs, regs[18])); + DEFINE(S_X20, offsetof(struct pt_regs, regs[20])); + DEFINE(S_X22, offsetof(struct pt_regs, regs[22])); + DEFINE(S_X24, offsetof(struct pt_regs, regs[24])); + DEFINE(S_X26, offsetof(struct pt_regs, regs[26])); + DEFINE(S_X28, offsetof(struct pt_regs, regs[28])); DEFINE(S_LR, offsetof(struct pt_regs, regs[30])); DEFINE(S_SP, offsetof(struct pt_regs, sp)); #ifdef CONFIG_COMPAT @@ -76,6 +98,7 @@ BLANK(); DEFINE(CLOCK_REALTIME, CLOCK_REALTIME); DEFINE(CLOCK_MONOTONIC, CLOCK_MONOTONIC); + DEFINE(CLOCK_MONOTONIC_RAW, CLOCK_MONOTONIC_RAW); DEFINE(CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC); DEFINE(CLOCK_REALTIME_COARSE, CLOCK_REALTIME_COARSE); DEFINE(CLOCK_MONOTONIC_COARSE,CLOCK_MONOTONIC_COARSE); @@ -83,6 +106,8 @@ DEFINE(NSEC_PER_SEC, NSEC_PER_SEC); BLANK(); DEFINE(VDSO_CS_CYCLE_LAST, offsetof(struct vdso_data, cs_cycle_last)); + DEFINE(VDSO_RAW_TIME_SEC, offsetof(struct vdso_data, raw_time_sec)); + DEFINE(VDSO_RAW_TIME_NSEC, offsetof(struct vdso_data, raw_time_nsec)); DEFINE(VDSO_XTIME_CLK_SEC, offsetof(struct vdso_data, xtime_clock_sec)); DEFINE(VDSO_XTIME_CLK_NSEC, offsetof(struct vdso_data, xtime_clock_nsec)); DEFINE(VDSO_XTIME_CRS_SEC, offsetof(struct vdso_data, xtime_coarse_sec)); @@ -90,7 +115,8 @@ DEFINE(VDSO_WTM_CLK_SEC, offsetof(struct vdso_data, wtm_clock_sec)); DEFINE(VDSO_WTM_CLK_NSEC, offsetof(struct vdso_data, wtm_clock_nsec)); DEFINE(VDSO_TB_SEQ_COUNT, offsetof(struct vdso_data, tb_seq_count)); - DEFINE(VDSO_CS_MULT, offsetof(struct vdso_data, cs_mult)); + DEFINE(VDSO_CS_MONO_MULT, offsetof(struct vdso_data, cs_mono_mult)); + DEFINE(VDSO_CS_RAW_MULT, offsetof(struct vdso_data, cs_raw_mult)); DEFINE(VDSO_CS_SHIFT, offsetof(struct vdso_data, cs_shift)); DEFINE(VDSO_TZ_MINWEST, offsetof(struct vdso_data, tz_minuteswest)); DEFINE(VDSO_TZ_DSTTIME, offsetof(struct vdso_data, tz_dsttime)); @@ -104,63 +130,39 @@ DEFINE(TZ_MINWEST, offsetof(struct timezone, tz_minuteswest)); DEFINE(TZ_DSTTIME, offsetof(struct timezone, tz_dsttime)); BLANK(); +#ifdef CONFIG_THREAD_INFO_IN_TASK + DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack)); + DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task)); + BLANK(); +#endif #ifdef CONFIG_KVM_ARM_HOST DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt)); DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs)); DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_regs, regs)); DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs)); - DEFINE(CPU_SP_EL1, offsetof(struct kvm_regs, sp_el1)); - DEFINE(CPU_ELR_EL1, offsetof(struct kvm_regs, elr_el1)); - DEFINE(CPU_SPSR, offsetof(struct kvm_regs, spsr)); - DEFINE(CPU_SYSREGS, offsetof(struct kvm_cpu_context, sys_regs)); + DEFINE(VCPU_FPEXC32_EL2, offsetof(struct kvm_vcpu, arch.ctxt.sys_regs[FPEXC32_EL2])); DEFINE(VCPU_ESR_EL2, offsetof(struct kvm_vcpu, arch.fault.esr_el2)); DEFINE(VCPU_FAR_EL2, offsetof(struct kvm_vcpu, arch.fault.far_el2)); DEFINE(VCPU_HPFAR_EL2, offsetof(struct kvm_vcpu, arch.fault.hpfar_el2)); - DEFINE(VCPU_DEBUG_FLAGS, offsetof(struct kvm_vcpu, arch.debug_flags)); - DEFINE(VCPU_DEBUG_PTR, offsetof(struct kvm_vcpu, arch.debug_ptr)); - DEFINE(DEBUG_BCR, offsetof(struct kvm_guest_debug_arch, dbg_bcr)); - DEFINE(DEBUG_BVR, offsetof(struct kvm_guest_debug_arch, dbg_bvr)); - DEFINE(DEBUG_WCR, offsetof(struct kvm_guest_debug_arch, dbg_wcr)); - DEFINE(DEBUG_WVR, offsetof(struct kvm_guest_debug_arch, dbg_wvr)); - DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2)); - DEFINE(VCPU_MDCR_EL2, offsetof(struct kvm_vcpu, arch.mdcr_el2)); - DEFINE(VCPU_IRQ_LINES, offsetof(struct kvm_vcpu, arch.irq_lines)); DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context)); - DEFINE(VCPU_HOST_DEBUG_STATE, offsetof(struct kvm_vcpu, arch.host_debug_state)); - DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl)); - DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval)); - DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff)); - DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled)); - DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm)); - DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu)); - DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr)); - DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr)); - DEFINE(VGIC_V2_CPU_MISR, offsetof(struct vgic_cpu, vgic_v2.vgic_misr)); - DEFINE(VGIC_V2_CPU_EISR, offsetof(struct vgic_cpu, vgic_v2.vgic_eisr)); - DEFINE(VGIC_V2_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_v2.vgic_elrsr)); - DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr)); - DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr)); - DEFINE(VGIC_V3_CPU_SRE, offsetof(struct vgic_cpu, vgic_v3.vgic_sre)); - DEFINE(VGIC_V3_CPU_HCR, offsetof(struct vgic_cpu, vgic_v3.vgic_hcr)); - DEFINE(VGIC_V3_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v3.vgic_vmcr)); - DEFINE(VGIC_V3_CPU_MISR, offsetof(struct vgic_cpu, vgic_v3.vgic_misr)); - DEFINE(VGIC_V3_CPU_EISR, offsetof(struct vgic_cpu, vgic_v3.vgic_eisr)); - DEFINE(VGIC_V3_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_v3.vgic_elrsr)); - DEFINE(VGIC_V3_CPU_AP0R, offsetof(struct vgic_cpu, vgic_v3.vgic_ap0r)); - DEFINE(VGIC_V3_CPU_AP1R, offsetof(struct vgic_cpu, vgic_v3.vgic_ap1r)); - DEFINE(VGIC_V3_CPU_LR, offsetof(struct vgic_cpu, vgic_v3.vgic_lr)); - DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr)); - DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); - DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); #endif #ifdef CONFIG_CPU_PM DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx)); DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask)); DEFINE(MPIDR_HASH_SHIFTS, offsetof(struct mpidr_hash, shift_aff)); - DEFINE(SLEEP_SAVE_SP_SZ, sizeof(struct sleep_save_sp)); - DEFINE(SLEEP_SAVE_SP_PHYS, offsetof(struct sleep_save_sp, save_ptr_stash_phys)); - DEFINE(SLEEP_SAVE_SP_VIRT, offsetof(struct sleep_save_sp, save_ptr_stash)); + DEFINE(SLEEP_STACK_DATA_SYSTEM_REGS, offsetof(struct sleep_stack_data, system_regs)); + DEFINE(SLEEP_STACK_DATA_CALLEE_REGS, offsetof(struct sleep_stack_data, callee_saved_regs)); +#endif + DEFINE(ARM_SMCCC_RES_X0_OFFS, offsetof(struct arm_smccc_res, a0)); + DEFINE(ARM_SMCCC_RES_X2_OFFS, offsetof(struct arm_smccc_res, a2)); + BLANK(); + DEFINE(HIBERN_PBE_ORIG, offsetof(struct pbe, orig_address)); + DEFINE(HIBERN_PBE_ADDR, offsetof(struct pbe, address)); + DEFINE(HIBERN_PBE_NEXT, offsetof(struct pbe, next)); + BLANK(); +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 + DEFINE(TRAMP_VALIAS, TRAMP_VALIAS); #endif return 0; } diff -ruw linux-4.4.115/arch/arm64/kernel/cpu_errata.c linux-4.4.115-fbx/arch/arm64/kernel/cpu_errata.c --- linux-4.4.115/arch/arm64/kernel/cpu_errata.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/cpu_errata.c 2019-01-22 16:16:21.547228694 +0100 @@ -21,32 +21,168 @@ #include #include -#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) -#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) -#define MIDR_THUNDERX MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) - -#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \ - MIDR_ARCHITECTURE_MASK) - static bool __maybe_unused is_affected_midr_range(const struct arm64_cpu_capabilities *entry) { - u32 midr = read_cpuid_id(); + return MIDR_IS_CPU_MODEL_RANGE(read_cpuid_id(), entry->midr_model, + entry->midr_range_min, + entry->midr_range_max); +} + +static bool __maybe_unused +is_kryo_midr(const struct arm64_cpu_capabilities *entry) +{ + u32 model; + + model = read_cpuid_id(); + model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) | + MIDR_ARCHITECTURE_MASK; + + return model == entry->midr_model; +} + +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR +#include +#include + +DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); + +#ifdef CONFIG_KVM +extern char __psci_hyp_bp_inval_start[], __psci_hyp_bp_inval_end[]; +extern char __qcom_hyp_sanitize_link_stack_start[]; +extern char __qcom_hyp_sanitize_link_stack_end[]; + +static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, + const char *hyp_vecs_end) +{ + void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K); + int i; + + for (i = 0; i < SZ_2K; i += 0x80) + memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start); + + flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K); +} + +static void __install_bp_hardening_cb(bp_hardening_cb_t fn, + const char *hyp_vecs_start, + const char *hyp_vecs_end) +{ + static int last_slot = -1; + static DEFINE_SPINLOCK(bp_lock); + int cpu, slot = -1; + + spin_lock(&bp_lock); + for_each_possible_cpu(cpu) { + if (per_cpu(bp_hardening_data.fn, cpu) == fn) { + slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu); + break; + } + } + + if (slot == -1) { + last_slot++; + BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start) + / SZ_2K) <= last_slot); + slot = last_slot; + __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end); + } + + __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot); + __this_cpu_write(bp_hardening_data.fn, fn); + spin_unlock(&bp_lock); +} +#else +#define __psci_hyp_bp_inval_start NULL +#define __psci_hyp_bp_inval_end NULL +#define __qcom_hyp_sanitize_link_stack_start NULL +#define __qcom_hyp_sanitize_link_stack_end NULL + +static void __maybe_unused __install_bp_hardening_cb(bp_hardening_cb_t fn, + const char *hyp_vecs_start, + const char *hyp_vecs_end) +{ + __this_cpu_write(bp_hardening_data.fn, fn); +} +#endif /* CONFIG_KVM */ - if ((midr & CPU_MODEL_MASK) != entry->midr_model) - return false; +static void __maybe_unused install_bp_hardening_cb( + const struct arm64_cpu_capabilities *entry, + bp_hardening_cb_t fn, + const char *hyp_vecs_start, + const char *hyp_vecs_end) +{ + u64 pfr0; - midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK; + if (!entry->matches(entry)) + return; - return (midr >= entry->midr_range_min && midr <= entry->midr_range_max); + pfr0 = read_cpuid(SYS_ID_AA64PFR0_EL1); + if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT)) + return; + + __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end); } +#include + +static int enable_psci_bp_hardening(void *data) +{ + const struct arm64_cpu_capabilities *entry = data; + + if (psci_ops.get_version) + install_bp_hardening_cb(entry, + (bp_hardening_cb_t)psci_ops.get_version, + __psci_hyp_bp_inval_start, + __psci_hyp_bp_inval_end); + + return 0; +} + +static void __maybe_unused qcom_link_stack_sanitization(void) +{ + u64 tmp; + + asm volatile("mov %0, x30 \n" + ".rept 16 \n" + "bl . + 4 \n" + ".endr \n" + "mov x30, %0 \n" + : "=&r" (tmp)); +} + +static void __maybe_unused qcom_bp_hardening(void) +{ + qcom_link_stack_sanitization(); + if (psci_ops.get_version) + psci_ops.get_version(); +} + +static int __maybe_unused enable_qcom_bp_hardening(void *data) +{ + const struct arm64_cpu_capabilities *entry = data; + + install_bp_hardening_cb(entry, + (bp_hardening_cb_t)qcom_bp_hardening, + __psci_hyp_bp_inval_start, + __psci_hyp_bp_inval_end); + return 0; +} + +#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */ + #define MIDR_RANGE(model, min, max) \ .matches = is_affected_midr_range, \ .midr_model = model, \ .midr_range_min = min, \ .midr_range_max = max +#define MIDR_ALL_VERSIONS(model) \ + .matches = is_affected_midr_range, \ + .midr_model = model, \ + .midr_range_min = 0, \ + .midr_range_max = (MIDR_VARIANT_MASK | MIDR_REVISION_MASK) + const struct arm64_cpu_capabilities arm64_errata[] = { #if defined(CONFIG_ARM64_ERRATUM_826319) || \ defined(CONFIG_ARM64_ERRATUM_827319) || \ @@ -91,6 +227,12 @@ .capability = ARM64_WORKAROUND_845719, MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04), }, + { + /* Kryo2xx Silver rAp4 */ + .desc = "Kryo2xx Silver erratum 845719", + .capability = ARM64_WORKAROUND_845719, + MIDR_RANGE(MIDR_KRYO2XX_SILVER, 0xA00004, 0xA00004), + }, #endif #ifdef CONFIG_CAVIUM_ERRATUM_23154 { @@ -109,6 +251,39 @@ (1 << MIDR_VARIANT_SHIFT) | 1), }, #endif +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR + { + .capability = ARM64_HARDEN_BRANCH_PREDICTOR, + MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), + .enable = enable_psci_bp_hardening, + }, + { + .capability = ARM64_HARDEN_BRANCH_PREDICTOR, + MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), + .enable = enable_psci_bp_hardening, + }, + { + .capability = ARM64_HARDEN_BRANCH_PREDICTOR, + MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), + .enable = enable_psci_bp_hardening, + }, + { + .capability = ARM64_HARDEN_BRANCH_PREDICTOR, + MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), + .enable = enable_psci_bp_hardening, + }, + { + .capability = ARM64_HARDEN_BRANCH_PREDICTOR, + MIDR_ALL_VERSIONS(MIDR_KRYO2XX_GOLD), + .enable = enable_psci_bp_hardening, + }, + { + .capability = ARM64_HARDEN_BRANCH_PREDICTOR, + .midr_model = MIDR_QCOM_KRYO, + .matches = is_kryo_midr, + .enable = enable_qcom_bp_hardening, + }, +#endif { } }; @@ -117,3 +292,8 @@ { update_cpu_capabilities(arm64_errata, "enabling workaround for"); } + +void __init enable_errata_workarounds(void) +{ + enable_cpu_capabilities(arm64_errata); +} diff -ruw linux-4.4.115/arch/arm64/kernel/cpufeature.c linux-4.4.115-fbx/arch/arm64/kernel/cpufeature.c --- linux-4.4.115/arch/arm64/kernel/cpufeature.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/cpufeature.c 2019-10-29 09:26:23.001196895 +0100 @@ -23,11 +23,13 @@ #include #include #include +#include #include #include #include #include #include +#include unsigned long elf_hwcap __read_mostly; EXPORT_SYMBOL_GPL(elf_hwcap); @@ -45,6 +47,7 @@ #endif DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); +EXPORT_SYMBOL(cpu_hwcaps); #define __ARM64_FTR_BITS(SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ { \ @@ -69,6 +72,10 @@ .width = 0, \ } +/* meta feature for alternatives */ +static bool __maybe_unused +cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry); + static struct arm64_ftr_bits ftr_id_aa64isar0[] = { ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0), ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64ISAR0_RDM_SHIFT, 4, 0), @@ -85,6 +92,7 @@ static struct arm64_ftr_bits ftr_id_aa64pfr0[] = { ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0), ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 4, 0), + ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI), ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI), @@ -125,6 +133,11 @@ ARM64_FTR_END, }; +static struct arm64_ftr_bits ftr_id_aa64mmfr2[] = { + ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_UAO_SHIFT, 4, 0), + ARM64_FTR_END, +}; + static struct arm64_ftr_bits ftr_ctr[] = { U_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RAO */ ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 3, 0), @@ -286,6 +299,7 @@ /* Op1 = 0, CRn = 0, CRm = 7 */ ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0), ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1), + ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2), /* Op1 = 3, CRn = 0, CRm = 0 */ ARM64_FTR_REG(SYS_CTR_EL0, ftr_ctr), @@ -410,6 +424,7 @@ init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1); init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0); init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1); + init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2); init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0); init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1); init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); @@ -519,6 +534,8 @@ info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0); taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu, info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1); + taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu, + info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2); /* * EL3 is not our concern. @@ -623,6 +640,56 @@ return has_sre; } +static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry) +{ + u32 midr = read_cpuid_id(); + u32 rv_min, rv_max; + + /* Cavium ThunderX pass 1.x and 2.x */ + rv_min = 0; + rv_max = (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK; + + return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, rv_min, rv_max); +} + +static bool runs_at_el2(const struct arm64_cpu_capabilities *entry) +{ + return is_kernel_in_hyp_mode(); +} + +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */ + +static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry) +{ + /* Forced on command line? */ + if (__kpti_forced) { + pr_info_once("kernel page table isolation forced %s by command line option\n", + __kpti_forced > 0 ? "ON" : "OFF"); + return __kpti_forced > 0; + } + + /* Useful for KASLR robustness */ + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) + return true; + + return false; +} + +static int __init parse_kpti(char *str) +{ + bool enabled; + int ret = strtobool(str, &enabled); + + if (ret) + return ret; + + __kpti_forced = enabled ? 1 : -1; + return 0; +} +__setup("kpti=", parse_kpti); +#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ + static const struct arm64_cpu_capabilities arm64_features[] = { { .desc = "GIC system register CPU interface", @@ -653,6 +720,39 @@ .min_field_value = 2, }, #endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */ + { + .desc = "Software prefetching using PRFM", + .capability = ARM64_HAS_NO_HW_PREFETCH, + .matches = has_no_hw_prefetch, + }, +#ifdef CONFIG_ARM64_UAO + { + .desc = "User Access Override", + .capability = ARM64_HAS_UAO, + .matches = has_cpuid_feature, + .sys_reg = SYS_ID_AA64MMFR2_EL1, + .field_pos = ID_AA64MMFR2_UAO_SHIFT, + .min_field_value = 1, + .enable = cpu_enable_uao, + }, +#endif /* CONFIG_ARM64_UAO */ +#ifdef CONFIG_ARM64_PAN + { + .capability = ARM64_ALT_PAN_NOT_UAO, + .matches = cpufeature_pan_not_uao, + }, +#endif /* CONFIG_ARM64_PAN */ + { + .desc = "Virtualization Host Extensions", + .capability = ARM64_HAS_VIRT_HOST_EXTN, + .matches = runs_at_el2, + }, +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 + { + .capability = ARM64_UNMAP_KERNEL_AT_EL0, + .matches = unmap_kernel_at_el0, + }, +#endif {}, }; @@ -686,7 +786,7 @@ {}, }; -static void cap_set_hwcap(const struct arm64_cpu_capabilities *cap) +static void __init cap_set_hwcap(const struct arm64_cpu_capabilities *cap) { switch (cap->hwcap_type) { case CAP_HWCAP: @@ -731,12 +831,12 @@ return rc; } -static void setup_cpu_hwcaps(void) +static void __init setup_cpu_hwcaps(void) { int i; const struct arm64_cpu_capabilities *hwcaps = arm64_hwcaps; - for (i = 0; hwcaps[i].desc; i++) + for (i = 0; hwcaps[i].matches; i++) if (hwcaps[i].matches(&hwcaps[i])) cap_set_hwcap(&hwcaps[i]); } @@ -746,11 +846,11 @@ { int i; - for (i = 0; caps[i].desc; i++) { + for (i = 0; caps[i].matches; i++) { if (!caps[i].matches(&caps[i])) continue; - if (!cpus_have_cap(caps[i].capability)) + if (!cpus_have_cap(caps[i].capability) && caps[i].desc) pr_info("%s %s\n", info, caps[i].desc); cpus_set_cap(caps[i].capability); } @@ -760,11 +860,11 @@ * Run through the enabled capabilities and enable() it on all active * CPUs */ -static void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) +void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) { int i; - for (i = 0; caps[i].desc; i++) + for (i = 0; caps[i].matches; i++) if (caps[i].enable && cpus_have_cap(caps[i].capability)) /* * Use stop_machine() as it schedules the work allowing @@ -772,7 +872,8 @@ * uses an IPI, giving us a PSTATE that disappears when * we return. */ - stop_machine(caps[i].enable, NULL, cpu_online_mask); + stop_machine(caps[i].enable, (void *)&caps[i], + cpu_online_mask); } #ifdef CONFIG_HOTPLUG_CPU @@ -798,35 +899,36 @@ static u64 __raw_read_system_reg(u32 sys_id) { switch (sys_id) { - case SYS_ID_PFR0_EL1: return (u64)read_cpuid(ID_PFR0_EL1); - case SYS_ID_PFR1_EL1: return (u64)read_cpuid(ID_PFR1_EL1); - case SYS_ID_DFR0_EL1: return (u64)read_cpuid(ID_DFR0_EL1); - case SYS_ID_MMFR0_EL1: return (u64)read_cpuid(ID_MMFR0_EL1); - case SYS_ID_MMFR1_EL1: return (u64)read_cpuid(ID_MMFR1_EL1); - case SYS_ID_MMFR2_EL1: return (u64)read_cpuid(ID_MMFR2_EL1); - case SYS_ID_MMFR3_EL1: return (u64)read_cpuid(ID_MMFR3_EL1); - case SYS_ID_ISAR0_EL1: return (u64)read_cpuid(ID_ISAR0_EL1); - case SYS_ID_ISAR1_EL1: return (u64)read_cpuid(ID_ISAR1_EL1); - case SYS_ID_ISAR2_EL1: return (u64)read_cpuid(ID_ISAR2_EL1); - case SYS_ID_ISAR3_EL1: return (u64)read_cpuid(ID_ISAR3_EL1); - case SYS_ID_ISAR4_EL1: return (u64)read_cpuid(ID_ISAR4_EL1); - case SYS_ID_ISAR5_EL1: return (u64)read_cpuid(ID_ISAR4_EL1); - case SYS_MVFR0_EL1: return (u64)read_cpuid(MVFR0_EL1); - case SYS_MVFR1_EL1: return (u64)read_cpuid(MVFR1_EL1); - case SYS_MVFR2_EL1: return (u64)read_cpuid(MVFR2_EL1); - - case SYS_ID_AA64PFR0_EL1: return (u64)read_cpuid(ID_AA64PFR0_EL1); - case SYS_ID_AA64PFR1_EL1: return (u64)read_cpuid(ID_AA64PFR0_EL1); - case SYS_ID_AA64DFR0_EL1: return (u64)read_cpuid(ID_AA64DFR0_EL1); - case SYS_ID_AA64DFR1_EL1: return (u64)read_cpuid(ID_AA64DFR0_EL1); - case SYS_ID_AA64MMFR0_EL1: return (u64)read_cpuid(ID_AA64MMFR0_EL1); - case SYS_ID_AA64MMFR1_EL1: return (u64)read_cpuid(ID_AA64MMFR1_EL1); - case SYS_ID_AA64ISAR0_EL1: return (u64)read_cpuid(ID_AA64ISAR0_EL1); - case SYS_ID_AA64ISAR1_EL1: return (u64)read_cpuid(ID_AA64ISAR1_EL1); - - case SYS_CNTFRQ_EL0: return (u64)read_cpuid(CNTFRQ_EL0); - case SYS_CTR_EL0: return (u64)read_cpuid(CTR_EL0); - case SYS_DCZID_EL0: return (u64)read_cpuid(DCZID_EL0); + case SYS_ID_PFR0_EL1: return read_cpuid(SYS_ID_PFR0_EL1); + case SYS_ID_PFR1_EL1: return read_cpuid(SYS_ID_PFR1_EL1); + case SYS_ID_DFR0_EL1: return read_cpuid(SYS_ID_DFR0_EL1); + case SYS_ID_MMFR0_EL1: return read_cpuid(SYS_ID_MMFR0_EL1); + case SYS_ID_MMFR1_EL1: return read_cpuid(SYS_ID_MMFR1_EL1); + case SYS_ID_MMFR2_EL1: return read_cpuid(SYS_ID_MMFR2_EL1); + case SYS_ID_MMFR3_EL1: return read_cpuid(SYS_ID_MMFR3_EL1); + case SYS_ID_ISAR0_EL1: return read_cpuid(SYS_ID_ISAR0_EL1); + case SYS_ID_ISAR1_EL1: return read_cpuid(SYS_ID_ISAR1_EL1); + case SYS_ID_ISAR2_EL1: return read_cpuid(SYS_ID_ISAR2_EL1); + case SYS_ID_ISAR3_EL1: return read_cpuid(SYS_ID_ISAR3_EL1); + case SYS_ID_ISAR4_EL1: return read_cpuid(SYS_ID_ISAR4_EL1); + case SYS_ID_ISAR5_EL1: return read_cpuid(SYS_ID_ISAR4_EL1); + case SYS_MVFR0_EL1: return read_cpuid(SYS_MVFR0_EL1); + case SYS_MVFR1_EL1: return read_cpuid(SYS_MVFR1_EL1); + case SYS_MVFR2_EL1: return read_cpuid(SYS_MVFR2_EL1); + + case SYS_ID_AA64PFR0_EL1: return read_cpuid(SYS_ID_AA64PFR0_EL1); + case SYS_ID_AA64PFR1_EL1: return read_cpuid(SYS_ID_AA64PFR0_EL1); + case SYS_ID_AA64DFR0_EL1: return read_cpuid(SYS_ID_AA64DFR0_EL1); + case SYS_ID_AA64DFR1_EL1: return read_cpuid(SYS_ID_AA64DFR0_EL1); + case SYS_ID_AA64MMFR0_EL1: return read_cpuid(SYS_ID_AA64MMFR0_EL1); + case SYS_ID_AA64MMFR1_EL1: return read_cpuid(SYS_ID_AA64MMFR1_EL1); + case SYS_ID_AA64MMFR2_EL1: return read_cpuid(SYS_ID_AA64MMFR2_EL1); + case SYS_ID_AA64ISAR0_EL1: return read_cpuid(SYS_ID_AA64ISAR0_EL1); + case SYS_ID_AA64ISAR1_EL1: return read_cpuid(SYS_ID_AA64ISAR1_EL1); + + case SYS_CNTFRQ_EL0: return read_cpuid(SYS_CNTFRQ_EL0); + case SYS_CTR_EL0: return read_cpuid(SYS_CTR_EL0); + case SYS_DCZID_EL0: return read_cpuid(SYS_DCZID_EL0); default: BUG(); return 0; @@ -876,7 +978,7 @@ return; caps = arm64_features; - for (i = 0; caps[i].desc; i++) { + for (i = 0; caps[i].matches; i++) { if (!cpus_have_cap(caps[i].capability) || !caps[i].sys_reg) continue; /* @@ -886,10 +988,10 @@ if (!feature_matches(__raw_read_system_reg(caps[i].sys_reg), &caps[i])) fail_incapable_cpu("arm64_features", &caps[i]); if (caps[i].enable) - caps[i].enable(NULL); + caps[i].enable((void *)&caps[i]); } - for (i = 0, caps = arm64_hwcaps; caps[i].desc; i++) { + for (i = 0, caps = arm64_hwcaps; caps[i].matches; i++) { if (!cpus_have_hwcap(&caps[i])) continue; if (!feature_matches(__raw_read_system_reg(caps[i].sys_reg), &caps[i])) @@ -905,7 +1007,7 @@ #endif /* CONFIG_HOTPLUG_CPU */ -static void setup_feature_capabilities(void) +static void __init setup_feature_capabilities(void) { update_cpu_capabilities(arm64_features, "detected feature:"); enable_cpu_capabilities(arm64_features); @@ -918,6 +1020,7 @@ /* Set the CPU feature capabilies */ setup_feature_capabilities(); + enable_errata_workarounds(); setup_cpu_hwcaps(); /* Advertise that we have computed the system capabilities */ @@ -931,7 +1034,13 @@ if (!cwg) pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n", cls); - if (L1_CACHE_BYTES < cls) - pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n", - L1_CACHE_BYTES, cls); + if (ARCH_DMA_MINALIGN < cls) + pr_warn("ARCH_DMA_MINALIGN smaller than the Cache Writeback Granule (%d < %d)\n", + ARCH_DMA_MINALIGN, cls); +} + +static bool __maybe_unused +cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry) +{ + return (cpus_have_cap(ARM64_HAS_PAN) && !cpus_have_cap(ARM64_HAS_UAO)); } diff -ruw linux-4.4.115/arch/arm64/kernel/cpuinfo.c linux-4.4.115-fbx/arch/arm64/kernel/cpuinfo.c --- linux-4.4.115/arch/arm64/kernel/cpuinfo.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/cpuinfo.c 2019-01-22 16:16:21.547228694 +0100 @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,10 @@ #include #include #include +#include + +char* (*arch_read_hardware_id)(void); +EXPORT_SYMBOL(arch_read_hardware_id); /* * In case the boot CPU is hotpluggable, we record its initial state and @@ -106,7 +111,9 @@ int i, j; bool compat = personality(current->personality) == PER_LINUX32; - for_each_online_cpu(i) { + seq_printf(m, "Processor\t: AArch64 Processor rev %d (%s)\n", + read_cpuid_id() & 15, ELF_PLATFORM); + for_each_present_cpu(i) { struct cpuinfo_arm64 *cpuinfo = &per_cpu(cpu_data, i); u32 midr = cpuinfo->reg_midr; @@ -156,6 +163,11 @@ seq_printf(m, "CPU revision\t: %d\n\n", MIDR_REVISION(midr)); } + if (!arch_read_hardware_id) + seq_printf(m, "Hardware\t: %s\n", machine_name); + else + seq_printf(m, "Hardware\t: %s\n", arch_read_hardware_id()); + return 0; } @@ -201,42 +213,48 @@ if (l1ip == ICACHE_POLICY_AIVIVT) set_bit(ICACHEF_AIVIVT, &__icache_flags); - pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); + pr_debug("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); } static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) { info->reg_cntfrq = arch_timer_get_cntfrq(); info->reg_ctr = read_cpuid_cachetype(); - info->reg_dczid = read_cpuid(DCZID_EL0); + info->reg_dczid = read_cpuid(SYS_DCZID_EL0); info->reg_midr = read_cpuid_id(); - info->reg_id_aa64dfr0 = read_cpuid(ID_AA64DFR0_EL1); - info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1); - info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1); - info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1); - info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); - info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); - info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1); - info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1); - - info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1); - info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1); - info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1); - info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1); - info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1); - info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1); - info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1); - info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1); - info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1); - info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1); - info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1); - info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1); - info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1); - - info->reg_mvfr0 = read_cpuid(MVFR0_EL1); - info->reg_mvfr1 = read_cpuid(MVFR1_EL1); - info->reg_mvfr2 = read_cpuid(MVFR2_EL1); + info->reg_id_aa64dfr0 = read_cpuid(SYS_ID_AA64DFR0_EL1); + info->reg_id_aa64dfr1 = read_cpuid(SYS_ID_AA64DFR1_EL1); + info->reg_id_aa64isar0 = read_cpuid(SYS_ID_AA64ISAR0_EL1); + info->reg_id_aa64isar1 = read_cpuid(SYS_ID_AA64ISAR1_EL1); + /* + * Explicitly mask out 16KB granule since we donot + * want to support it + */ + info->reg_id_aa64mmfr0 = read_cpuid(SYS_ID_AA64MMFR0_EL1) & + (~MMFR0_EL1_16KGRAN_MASK); + info->reg_id_aa64mmfr1 = read_cpuid(SYS_ID_AA64MMFR1_EL1); + info->reg_id_aa64mmfr2 = read_cpuid(SYS_ID_AA64MMFR2_EL1); + info->reg_id_aa64pfr0 = read_cpuid(SYS_ID_AA64PFR0_EL1); + info->reg_id_aa64pfr1 = read_cpuid(SYS_ID_AA64PFR1_EL1); + + info->reg_id_dfr0 = read_cpuid(SYS_ID_DFR0_EL1); + info->reg_id_isar0 = read_cpuid(SYS_ID_ISAR0_EL1); + info->reg_id_isar1 = read_cpuid(SYS_ID_ISAR1_EL1); + info->reg_id_isar2 = read_cpuid(SYS_ID_ISAR2_EL1); + info->reg_id_isar3 = read_cpuid(SYS_ID_ISAR3_EL1); + info->reg_id_isar4 = read_cpuid(SYS_ID_ISAR4_EL1); + info->reg_id_isar5 = read_cpuid(SYS_ID_ISAR5_EL1); + info->reg_id_mmfr0 = read_cpuid(SYS_ID_MMFR0_EL1); + info->reg_id_mmfr1 = read_cpuid(SYS_ID_MMFR1_EL1); + info->reg_id_mmfr2 = read_cpuid(SYS_ID_MMFR2_EL1); + info->reg_id_mmfr3 = read_cpuid(SYS_ID_MMFR3_EL1); + info->reg_id_pfr0 = read_cpuid(SYS_ID_PFR0_EL1); + info->reg_id_pfr1 = read_cpuid(SYS_ID_PFR1_EL1); + + info->reg_mvfr0 = read_cpuid(SYS_MVFR0_EL1); + info->reg_mvfr1 = read_cpuid(SYS_MVFR1_EL1); + info->reg_mvfr2 = read_cpuid(SYS_MVFR2_EL1); cpuinfo_detect_icache_policy(info); diff -ruw linux-4.4.115/arch/arm64/kernel/cpu_ops.c linux-4.4.115-fbx/arch/arm64/kernel/cpu_ops.c --- linux-4.4.115/arch/arm64/kernel/cpu_ops.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/cpu_ops.c 2019-10-29 09:26:23.001196895 +0100 @@ -25,19 +25,30 @@ #include extern const struct cpu_operations smp_spin_table_ops; +extern const struct cpu_operations acpi_parking_protocol_ops; extern const struct cpu_operations cpu_psci_ops; const struct cpu_operations *cpu_ops[NR_CPUS]; -static const struct cpu_operations *supported_cpu_ops[] __initconst = { +static const struct cpu_operations *dt_supported_cpu_ops[] __initconst = { &smp_spin_table_ops, &cpu_psci_ops, NULL, }; +static const struct cpu_operations *acpi_supported_cpu_ops[] __initconst = { +#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL + &acpi_parking_protocol_ops, +#endif + &cpu_psci_ops, + NULL, +}; + static const struct cpu_operations * __init cpu_get_ops(const char *name) { - const struct cpu_operations **ops = supported_cpu_ops; + const struct cpu_operations **ops; + + ops = acpi_disabled ? dt_supported_cpu_ops : acpi_supported_cpu_ops; while (*ops) { if (!strcmp(name, (*ops)->name)) @@ -75,9 +86,17 @@ } } else { enable_method = acpi_get_enable_method(cpu); - if (!enable_method) + if (!enable_method) { + /* + * In ACPI systems the boot CPU does not require + * checking the enable method since for some + * boot protocol (ie parking protocol) it need not + * be initialized. Don't warn spuriously. + */ + if (cpu != 0) pr_err("Unsupported ACPI enable-method\n"); } + } return enable_method; } diff -ruw linux-4.4.115/arch/arm64/kernel/debug-monitors.c linux-4.4.115-fbx/arch/arm64/kernel/debug-monitors.c --- linux-4.4.115/arch/arm64/kernel/debug-monitors.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/debug-monitors.c 2019-01-22 16:16:21.547228694 +0100 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -48,6 +49,7 @@ asm volatile("msr mdscr_el1, %0" :: "r" (mdscr)); local_dbg_restore(flags); } +NOKPROBE_SYMBOL(mdscr_write); static u32 mdscr_read(void) { @@ -55,6 +57,7 @@ asm volatile("mrs %0, mdscr_el1" : "=r" (mdscr)); return mdscr; } +NOKPROBE_SYMBOL(mdscr_read); /* * Allow root to disable self-hosted debug from userspace. @@ -103,6 +106,7 @@ mdscr_write(mdscr); } } +NOKPROBE_SYMBOL(enable_debug_monitors); void disable_debug_monitors(enum dbg_active_el el) { @@ -123,6 +127,7 @@ mdscr_write(mdscr); } } +NOKPROBE_SYMBOL(disable_debug_monitors); /* * OS lock clearing. @@ -173,6 +178,7 @@ spsr |= DBG_SPSR_SS; regs->pstate = spsr; } +NOKPROBE_SYMBOL(set_regs_spsr_ss); static void clear_regs_spsr_ss(struct pt_regs *regs) { @@ -182,6 +188,7 @@ spsr &= ~DBG_SPSR_SS; regs->pstate = spsr; } +NOKPROBE_SYMBOL(clear_regs_spsr_ss); /* EL1 Single Step Handler hooks */ static LIST_HEAD(step_hook); @@ -225,6 +232,7 @@ return retval; } +NOKPROBE_SYMBOL(call_step_hook); static int single_step_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs) @@ -253,6 +261,10 @@ */ user_rewind_single_step(current); } else { +#ifdef CONFIG_KPROBES + if (kprobe_single_step_handler(regs, esr) == DBG_HOOK_HANDLED) + return 0; +#endif if (call_step_hook(regs, esr) == DBG_HOOK_HANDLED) return 0; @@ -266,6 +278,7 @@ return 0; } +NOKPROBE_SYMBOL(single_step_handler); /* * Breakpoint handler is re-entrant as another breakpoint can @@ -303,6 +316,7 @@ return fn ? fn(regs, esr) : DBG_HOOK_ERROR; } +NOKPROBE_SYMBOL(call_break_hook); static int brk_handler(unsigned long addr, unsigned int esr, struct pt_regs *regs) @@ -318,13 +332,21 @@ }; force_sig_info(SIGTRAP, &info, current); - } else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) { - pr_warning("Unexpected kernel BRK exception at EL1\n"); + } +#ifdef CONFIG_KPROBES + else if ((esr & BRK64_ESR_MASK) == BRK64_ESR_KPROBES) { + if (kprobe_breakpoint_handler(regs, esr) != DBG_HOOK_HANDLED) + return -EFAULT; + } +#endif + else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) { + pr_warn("Unexpected kernel BRK exception at EL1\n"); return -EFAULT; } return 0; } +NOKPROBE_SYMBOL(brk_handler); int aarch32_break_handler(struct pt_regs *regs) { @@ -369,6 +391,7 @@ force_sig_info(SIGTRAP, &info, current); return 0; } +NOKPROBE_SYMBOL(aarch32_break_handler); static int __init debug_traps_init(void) { @@ -390,6 +413,7 @@ if (test_ti_thread_flag(task_thread_info(task), TIF_SINGLESTEP)) set_regs_spsr_ss(task_pt_regs(task)); } +NOKPROBE_SYMBOL(user_rewind_single_step); void user_fastforward_single_step(struct task_struct *task) { @@ -405,6 +429,7 @@ mdscr_write(mdscr_read() | DBG_MDSCR_SS); enable_debug_monitors(DBG_ACTIVE_EL1); } +NOKPROBE_SYMBOL(kernel_enable_single_step); void kernel_disable_single_step(void) { @@ -412,12 +437,14 @@ mdscr_write(mdscr_read() & ~DBG_MDSCR_SS); disable_debug_monitors(DBG_ACTIVE_EL1); } +NOKPROBE_SYMBOL(kernel_disable_single_step); int kernel_active_single_step(void) { WARN_ON(!irqs_disabled()); return mdscr_read() & DBG_MDSCR_SS; } +NOKPROBE_SYMBOL(kernel_active_single_step); /* ptrace API */ void user_enable_single_step(struct task_struct *task) @@ -427,8 +454,10 @@ if (!test_and_set_ti_thread_flag(ti, TIF_SINGLESTEP)) set_regs_spsr_ss(task_pt_regs(task)); } +NOKPROBE_SYMBOL(user_enable_single_step); void user_disable_single_step(struct task_struct *task) { clear_ti_thread_flag(task_thread_info(task), TIF_SINGLESTEP); } +NOKPROBE_SYMBOL(user_disable_single_step); diff -ruw linux-4.4.115/arch/arm64/kernel/entry-fpsimd.S linux-4.4.115-fbx/arch/arm64/kernel/entry-fpsimd.S --- linux-4.4.115/arch/arm64/kernel/entry-fpsimd.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/entry-fpsimd.S 2019-01-22 16:16:21.547228694 +0100 @@ -64,4 +64,20 @@ ret ENDPROC(fpsimd_load_partial_state) +#ifdef CONFIG_ENABLE_FP_SIMD_SETTINGS +ENTRY(fpsimd_enable_trap) + mrs x0, cpacr_el1 + bic x0, x0, #(3 << 20) + orr x0, x0, #(1 << 20) + msr cpacr_el1, x0 + ret +ENDPROC(fpsimd_enable_trap) +ENTRY(fpsimd_disable_trap) + mrs x0, cpacr_el1 + orr x0, x0, #(3 << 20) + msr cpacr_el1, x0 + ret +ENDPROC(fpsimd_disable_trap) +#endif + #endif diff -ruw linux-4.4.115/arch/arm64/kernel/entry.S linux-4.4.115-fbx/arch/arm64/kernel/entry.S --- linux-4.4.115/arch/arm64/kernel/entry.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/entry.S 2019-01-22 16:16:21.547228694 +0100 @@ -27,8 +27,12 @@ #include #include #include +#include #include +#include +#include #include +#include #include #include @@ -67,8 +71,31 @@ #define BAD_FIQ 2 #define BAD_ERROR 3 - .macro kernel_entry, el, regsize = 64 + .macro kernel_ventry, el, label, regsize = 64 + .align 7 +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +alternative_if ARM64_UNMAP_KERNEL_AT_EL0 + .if \el == 0 + .if \regsize == 64 + mrs x30, tpidrro_el0 + msr tpidrro_el0, xzr + .else + mov x30, xzr + .endif + .endif +alternative_else_nop_endif +#endif + sub sp, sp, #S_FRAME_SIZE + b el\()\el\()_\label + .endm + + .macro tramp_alias, dst, sym + mov_q \dst, TRAMP_VALIAS + add \dst, \dst, #(\sym - .entry.tramp.text) + .endm + + .macro kernel_entry, el, regsize = 64 .if \regsize == 32 mov w0, w0 // zero upper 32 bits of x0 .endif @@ -90,21 +117,64 @@ .if \el == 0 mrs x21, sp_el0 - get_thread_info tsk // Ensure MDSCR_EL1.SS is clear, +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr_this_cpu tsk, __entry_task, x20 // Ensure MDSCR_EL1.SS is clear, + ldr x19, [tsk, #TSK_TI_FLAGS] // since we can unmask debug +#else + mov tsk, sp + and tsk, tsk, #~(THREAD_SIZE - 1) // Ensure MDSCR_EL1.SS is clear, ldr x19, [tsk, #TI_FLAGS] // since we can unmask debug +#endif disable_step_tsk x19, x20 // exceptions when scheduling. + + mov x29, xzr // fp pointed to user-space .else add x21, sp, #S_FRAME_SIZE get_thread_info tsk /* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */ +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr x20, [tsk, #TSK_TI_ADDR_LIMIT] +#else ldr x20, [tsk, #TI_ADDR_LIMIT] +#endif str x20, [sp, #S_ORIG_ADDR_LIMIT] mov x20, #TASK_SIZE_64 +#ifdef CONFIG_THREAD_INFO_IN_TASK + str x20, [tsk, #TSK_TI_ADDR_LIMIT] +#else str x20, [tsk, #TI_ADDR_LIMIT] +#endif + ALTERNATIVE(nop, SET_PSTATE_UAO(0), ARM64_HAS_UAO, CONFIG_ARM64_UAO) .endif /* \el == 0 */ mrs x22, elr_el1 mrs x23, spsr_el1 stp lr, x21, [sp, #S_LR] + +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + /* + * Set the TTBR0 PAN bit in SPSR. When the exception is taken from + * EL0, there is no need to check the state of TTBR0_EL1 since + * accesses are always enabled. + * Note that the meaning of this bit differs from the ARMv8.1 PAN + * feature as all TTBR0_EL1 accesses are disabled, not just those to + * user mappings. + */ +alternative_if ARM64_HAS_PAN + b 1f // skip TTBR0 PAN +alternative_else_nop_endif + + .if \el != 0 + mrs x21, ttbr0_el1 + tst x21, #TTBR_ASID_MASK // Check for the reserved ASID + orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR + b.eq 1f // TTBR0 access already disabled + and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR + .endif + + __uaccess_ttbr0_disable x21 +1: +#endif + stp x22, x23, [sp, #S_PC] /* @@ -116,6 +186,13 @@ .endif /* + * Set sp_el0 to current thread_info. + */ + .if \el == 0 + msr sp_el0, tsk + .endif + + /* * Registers that may be useful after this macro is invoked: * * x21 - aborted SP @@ -128,33 +205,70 @@ .if \el != 0 /* Restore the task's original addr_limit. */ ldr x20, [sp, #S_ORIG_ADDR_LIMIT] +#ifdef CONFIG_THREAD_INFO_IN_TASK + str x20, [tsk, #TSK_TI_ADDR_LIMIT] +#else str x20, [tsk, #TI_ADDR_LIMIT] +#endif + + /* No need to restore UAO, it will be restored from SPSR_EL1 */ .endif ldp x21, x22, [sp, #S_PC] // load ELR, SPSR .if \el == 0 ct_user_enter + .endif + +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + /* + * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR + * PAN bit checking. + */ +alternative_if ARM64_HAS_PAN + b 2f // skip TTBR0 PAN +alternative_else_nop_endif + + .if \el != 0 + tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set + .endif + + __uaccess_ttbr0_enable x0, x1 + + .if \el == 0 + /* + * Enable errata workarounds only if returning to user. The only + * workaround currently required for TTBR0_EL1 changes are for the + * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache + * corruption). + */ + bl post_ttbr_update_workaround + .endif +1: + .if \el != 0 + and x22, x22, #~PSR_PAN_BIT // ARMv8.0 CPUs do not understand this bit + .endif +2: +#endif + + .if \el == 0 ldr x23, [sp, #S_SP] // load return stack pointer msr sp_el0, x23 + tst x22, #PSR_MODE32_BIT // native task? + b.eq 3f + #ifdef CONFIG_ARM64_ERRATUM_845719 -alternative_if_not ARM64_WORKAROUND_845719 - nop - nop -#ifdef CONFIG_PID_IN_CONTEXTIDR - nop -#endif -alternative_else - tbz x22, #4, 1f +alternative_if ARM64_WORKAROUND_845719 #ifdef CONFIG_PID_IN_CONTEXTIDR mrs x29, contextidr_el1 msr contextidr_el1, x29 #else msr contextidr_el1, xzr #endif -1: -alternative_endif +alternative_else_nop_endif #endif +3: .endif + msr elr_el1, x21 // set up the return data msr spsr_el1, x22 ldp x0, x1, [sp, #16 * 0] @@ -174,12 +288,65 @@ ldp x28, x29, [sp, #16 * 14] ldr lr, [sp, #S_LR] add sp, sp, #S_FRAME_SIZE // restore sp - eret // return to kernel + + .if \el == 0 +alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 + bne 4f + msr far_el1, x30 + tramp_alias x30, tramp_exit_native + br x30 +4: + tramp_alias x30, tramp_exit_compat + br x30 +#endif + .else + eret + .endif .endm - .macro get_thread_info, rd - mov \rd, sp - and \rd, \rd, #~(THREAD_SIZE - 1) // top of stack + .macro irq_stack_entry + mov x19, sp // preserve the original sp + + /* + * Compare sp with the base of the task stack. + * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack, + * and should switch to the irq stack. + */ +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr x25, [tsk, TSK_STACK] + eor x25, x25, x19 + and x25, x25, #~(THREAD_SIZE - 1) + cbnz x25, 9998f +#else + and x25, x19, #~(THREAD_SIZE - 1) + cmp x25, tsk + b.ne 9998f +#endif + + adr_this_cpu x25, irq_stack, x26 + mov x26, #IRQ_STACK_START_SP + add x26, x25, x26 + + /* switch to the irq stack */ + mov sp, x26 + + /* + * Add a dummy stack frame, this non-standard format is fixed up + * by unwind_frame() + */ + stp x29, x19, [sp, #-16]! + mov x29, sp + +9998: + .endm + + /* + * x19 should be preserved between irq_stack_entry and + * irq_stack_exit. + */ + .macro irq_stack_exit + mov sp, x19 .endm /* @@ -197,10 +364,11 @@ * Interrupt handling. */ .macro irq_handler - adrp x1, handle_arch_irq - ldr x1, [x1, #:lo12:handle_arch_irq] + ldr_l x1, handle_arch_irq mov x0, sp + irq_stack_entry blr x1 + irq_stack_exit .endm .text @@ -208,34 +376,35 @@ /* * Exception vectors. */ + .pushsection ".entry.text", "ax" .align 11 ENTRY(vectors) - ventry el1_sync_invalid // Synchronous EL1t - ventry el1_irq_invalid // IRQ EL1t - ventry el1_fiq_invalid // FIQ EL1t - ventry el1_error_invalid // Error EL1t - - ventry el1_sync // Synchronous EL1h - ventry el1_irq // IRQ EL1h - ventry el1_fiq_invalid // FIQ EL1h - ventry el1_error_invalid // Error EL1h - - ventry el0_sync // Synchronous 64-bit EL0 - ventry el0_irq // IRQ 64-bit EL0 - ventry el0_fiq_invalid // FIQ 64-bit EL0 - ventry el0_error_invalid // Error 64-bit EL0 + kernel_ventry 1, sync_invalid // Synchronous EL1t + kernel_ventry 1, irq_invalid // IRQ EL1t + kernel_ventry 1, fiq_invalid // FIQ EL1t + kernel_ventry 1, error_invalid // Error EL1t + + kernel_ventry 1, sync // Synchronous EL1h + kernel_ventry 1, irq // IRQ EL1h + kernel_ventry 1, fiq_invalid // FIQ EL1h + kernel_ventry 1, error_invalid // Error EL1h + + kernel_ventry 0, sync // Synchronous 64-bit EL0 + kernel_ventry 0, irq // IRQ 64-bit EL0 + kernel_ventry 0, fiq_invalid // FIQ 64-bit EL0 + kernel_ventry 0, error_invalid // Error 64-bit EL0 #ifdef CONFIG_COMPAT - ventry el0_sync_compat // Synchronous 32-bit EL0 - ventry el0_irq_compat // IRQ 32-bit EL0 - ventry el0_fiq_invalid_compat // FIQ 32-bit EL0 - ventry el0_error_invalid_compat // Error 32-bit EL0 -#else - ventry el0_sync_invalid // Synchronous 32-bit EL0 - ventry el0_irq_invalid // IRQ 32-bit EL0 - ventry el0_fiq_invalid // FIQ 32-bit EL0 - ventry el0_error_invalid // Error 32-bit EL0 + kernel_ventry 0, sync_compat, 32 // Synchronous 32-bit EL0 + kernel_ventry 0, irq_compat, 32 // IRQ 32-bit EL0 + kernel_ventry 0, fiq_invalid_compat, 32 // FIQ 32-bit EL0 + kernel_ventry 0, error_invalid_compat, 32 // Error 32-bit EL0 +#else + kernel_ventry 0, sync_invalid, 32 // Synchronous 32-bit EL0 + kernel_ventry 0, irq_invalid, 32 // IRQ 32-bit EL0 + kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0 + kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0 #endif END(vectors) @@ -243,7 +412,7 @@ * Invalid mode handlers */ .macro inv_entry, el, reason, regsize = 64 - kernel_entry el, \regsize + kernel_entry \el, \regsize mov x0, sp mov x1, #\reason mrs x2, esr_el1 @@ -297,11 +466,16 @@ */ .align 6 el1_sync: +#ifdef CONFIG_QCOM_TLB_EL2_HANDLER + smc #0xffff +#endif kernel_entry 1 mrs x1, esr_el1 // read the syndrome register lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1 b.eq el1_da + cmp x24, #ESR_ELx_EC_IABT_CUR // instruction abort in EL1 + b.eq el1_ia cmp x24, #ESR_ELx_EC_SYS64 // configurable trap b.eq el1_undef cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception @@ -313,6 +487,11 @@ cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1 b.ge el1_dbg b el1_inv + +el1_ia: + /* + * Fall through to the Data abort case + */ el1_da: /* * Data abort handling @@ -373,13 +552,21 @@ bl trace_hardirqs_off #endif + get_thread_info tsk irq_handler #ifdef CONFIG_PREEMPT - get_thread_info tsk +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr w24, [tsk, #TSK_TI_PREEMPT] // get preempt count +#else ldr w24, [tsk, #TI_PREEMPT] // get preempt count +#endif cbnz w24, 1f // preempt count != 0 +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr x0, [tsk, #TSK_TI_FLAGS] // get flags +#else ldr x0, [tsk, #TI_FLAGS] // get flags +#endif tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling? bl el1_preempt 1: @@ -394,7 +581,11 @@ el1_preempt: mov x24, lr 1: bl preempt_schedule_irq // irq en/disable is done inside +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr x0, [tsk, #TSK_TI_FLAGS] // get new tasks TI_FLAGS +#else ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS +#endif tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling? ret x24 #endif @@ -418,7 +609,7 @@ cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception b.eq el0_fpsimd_exc cmp x24, #ESR_ELx_EC_SYS64 // configurable trap - b.eq el0_undef + b.eq el0_sys cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception b.eq el0_sp_pc cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception @@ -442,7 +633,7 @@ cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0 b.eq el0_ia cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access - b.eq el0_fpsimd_acc + b.eq el0_fpsimd_acc_compat cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception b.eq el0_fpsimd_exc cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception @@ -495,13 +686,15 @@ * Instruction abort handling */ mrs x26, far_el1 - // enable interrupts before calling the main handler - enable_dbg_and_irq + enable_dbg +#ifdef CONFIG_TRACE_IRQFLAGS + bl trace_hardirqs_off +#endif ct_user_exit mov x0, x26 - orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts + mov x1, x25 mov x2, sp - bl do_mem_abort + bl do_el0_ia_bp_hardening b ret_to_user el0_fpsimd_acc: /* @@ -513,6 +706,17 @@ mov x1, sp bl do_fpsimd_acc b ret_to_user +el0_fpsimd_acc_compat: + /* + * Floating Point or Advanced SIMD access + */ + enable_dbg + ct_user_exit + mov x0, x25 + mov x1, sp + bl do_fpsimd_acc_compat + b ret_to_user + el0_fpsimd_exc: /* * Floating Point or Advanced SIMD exception @@ -546,6 +750,16 @@ mov x0, sp bl do_undefinstr b ret_to_user +el0_sys: + /* + * System instructions, for trapped cache maintenance instructions + */ + enable_dbg_and_irq + ct_user_exit + mov x0, x25 + mov x1, sp + bl do_sysinstr + b ret_to_user el0_dbg: /* * Debug exception handling @@ -613,7 +827,40 @@ ldp x27, x28, [x8], #16 ldp x29, x9, [x8], #16 ldr lr, [x8] +#ifdef CONFIG_ARM64_REG_REBALANCE_ON_CTX_SW + orr x13, x13, x13 + orr x14, x14, x14 + orr x15, x15, x15 + orr x16, x16, x16 + orr x17, x17, x17 + orr x18, x18, x18 + orr x19, x19, x19 + orr x20, x20, x20 + orr x21, x21, x21 + mov v0.16b, v0.16b + mov v1.16b, v1.16b + mov v2.16b, v2.16b + mov v3.16b, v3.16b + mov v4.16b, v4.16b + mov v5.16b, v5.16b + mov v6.16b, v6.16b + mov v7.16b, v7.16b + mov v8.16b, v8.16b + mov v9.16b, v9.16b + mov v10.16b, v10.16b + mov v11.16b, v11.16b + mov v12.16b, v12.16b + mov v13.16b, v13.16b + mov v14.16b, v14.16b + mov v15.16b, v15.16b +#endif mov sp, x9 +#ifdef CONFIG_THREAD_INFO_IN_TASK + msr sp_el0, x1 +#else + and x9, x9, #~(THREAD_SIZE - 1) + msr sp_el0, x9 +#endif ret ENDPROC(cpu_switch_to) @@ -624,7 +871,11 @@ ret_fast_syscall: disable_irq // disable interrupts str x0, [sp, #S_X0] // returned x0 +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for syscall tracing +#else ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing +#endif and x2, x1, #_TIF_SYSCALL_WORK cbnz x2, ret_fast_syscall_trace and x2, x1, #_TIF_WORK_MASK @@ -641,14 +892,14 @@ work_pending: tbnz x1, #TIF_NEED_RESCHED, work_resched /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */ - ldr x2, [sp, #S_PSTATE] mov x0, sp // 'regs' - tst x2, #PSR_MODE_MASK // user mode regs? - b.ne no_work_pending // returning to kernel enable_irq // enable interrupts for do_notify_resume() bl do_notify_resume b ret_to_user work_resched: +#ifdef CONFIG_TRACE_IRQFLAGS + bl trace_hardirqs_off // the IRQs are off here, inform the tracing code +#endif bl schedule /* @@ -656,11 +907,14 @@ */ ret_to_user: disable_irq // disable interrupts +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr x1, [tsk, #TSK_TI_FLAGS] +#else ldr x1, [tsk, #TI_FLAGS] +#endif and x2, x1, #_TIF_WORK_MASK cbnz x2, work_pending enable_step_tsk x1, x2 -no_work_pending: kernel_exit 0 ENDPROC(ret_to_user) @@ -689,7 +943,11 @@ enable_dbg_and_irq ct_user_exit 1 +#ifdef CONFIG_THREAD_INFO_IN_TASK + ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks +#else ldr x16, [tsk, #TI_FLAGS] // check for syscall hooks +#endif tst x16, #_TIF_SYSCALL_WORK b.ne __sys_trace cmp scno, sc_nr // check upper syscall limit @@ -740,6 +998,120 @@ bl do_ni_syscall b __sys_trace_return + .popsection // .entry.text +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +/* + * Exception vectors trampoline. + */ + .pushsection ".entry.tramp.text", "ax" + + .macro tramp_map_kernel, tmp + mrs \tmp, ttbr1_el1 + sub \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) + bic \tmp, \tmp, #USER_ASID_FLAG + msr ttbr1_el1, \tmp +#ifdef CONFIG_ARCH_MSM8996 + /* ASID already in \tmp[63:48] */ + movk \tmp, #:abs_g2_nc:(TRAMP_VALIAS >> 12) + movk \tmp, #:abs_g1_nc:(TRAMP_VALIAS >> 12) + /* 2MB boundary containing the vectors, so we nobble the walk cache */ + movk \tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12) + isb + tlbi vae1, \tmp + dsb nsh +#endif /* CONFIG_ARCH_MSM8996 */ + .endm + + .macro tramp_unmap_kernel, tmp + mrs \tmp, ttbr1_el1 + add \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) + orr \tmp, \tmp, #USER_ASID_FLAG + msr ttbr1_el1, \tmp + /* + * We avoid running the post_ttbr_update_workaround here because the + * user and kernel ASIDs don't have conflicting mappings, so any + * "blessing" as described in: + * + * http://lkml.kernel.org/r/56BB848A.6060603@caviumnetworks.com + * + * will not hurt correctness. Whilst this may partially defeat the + * point of using split ASIDs in the first place, it avoids + * the hit of invalidating the entire I-cache on every return to + * userspace. + */ + .endm + + .macro tramp_ventry, regsize = 64 + .align 7 +1: + .if \regsize == 64 + msr tpidrro_el0, x30 // Restored in kernel_ventry + .endif + bl 2f + b . +2: + tramp_map_kernel x30 +#ifdef CONFIG_RANDOMIZE_BASE + adr x30, tramp_vectors + PAGE_SIZE +#ifndef CONFIG_ARCH_MSM8996 + isb +#endif + ldr x30, [x30] +#else + ldr x30, =vectors +#endif + prfm plil1strm, [x30, #(1b - tramp_vectors)] + msr vbar_el1, x30 + add x30, x30, #(1b - tramp_vectors) + isb + ret + .endm + + .macro tramp_exit, regsize = 64 + adr x30, tramp_vectors + msr vbar_el1, x30 + tramp_unmap_kernel x30 + .if \regsize == 64 + mrs x30, far_el1 + .endif + eret + .endm + + .align 11 +ENTRY(tramp_vectors) + .space 0x400 + + tramp_ventry + tramp_ventry + tramp_ventry + tramp_ventry + + tramp_ventry 32 + tramp_ventry 32 + tramp_ventry 32 + tramp_ventry 32 +END(tramp_vectors) + +ENTRY(tramp_exit_native) + tramp_exit +END(tramp_exit_native) + +ENTRY(tramp_exit_compat) + tramp_exit 32 +END(tramp_exit_compat) + + .ltorg + .popsection // .entry.tramp.text +#ifdef CONFIG_RANDOMIZE_BASE + .pushsection ".rodata", "a" + .align PAGE_SHIFT + .globl __entry_tramp_data_start +__entry_tramp_data_start: + .quad vectors + .popsection // .rodata +#endif /* CONFIG_RANDOMIZE_BASE */ +#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ + /* * Special system call wrappers. */ diff -ruw linux-4.4.115/arch/arm64/kernel/fpsimd.c linux-4.4.115-fbx/arch/arm64/kernel/fpsimd.c --- linux-4.4.115/arch/arm64/kernel/fpsimd.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/fpsimd.c 2019-01-22 16:16:21.551228730 +0100 @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include +#include #define FPEXC_IOF (1 << 0) #define FPEXC_DZF (1 << 1) @@ -35,6 +37,8 @@ #define FPEXC_IXF (1 << 4) #define FPEXC_IDF (1 << 7) +#define FP_SIMD_BIT 31 + /* * In order to reduce the number of times the FPSIMD state is needlessly saved * and restored, we need to keep track of two things: @@ -88,14 +92,42 @@ * whatever is in the FPSIMD registers is not saved to memory, but discarded. */ static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state); +static DEFINE_PER_CPU(int, fpsimd_stg_enable); + +static int fpsimd_settings = 0x1; /* default = 0x1 */ +module_param(fpsimd_settings, int, 0644); + +void fpsimd_settings_enable(void) +{ + set_app_setting_bit(FP_SIMD_BIT); +} + +void fpsimd_settings_disable(void) +{ + clear_app_setting_bit(FP_SIMD_BIT); +} /* * Trapped FP/ASIMD access. */ void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) { - /* TODO: implement lazy context saving/restoring */ - WARN_ON(1); + if (!fpsimd_settings) + return; + + fpsimd_disable_trap(); + fpsimd_settings_disable(); + this_cpu_write(fpsimd_stg_enable, 0); +} + +void do_fpsimd_acc_compat(unsigned int esr, struct pt_regs *regs) +{ + if (!fpsimd_settings) + return; + + fpsimd_disable_trap(); + fpsimd_settings_enable(); + this_cpu_write(fpsimd_stg_enable, 1); } /* @@ -135,6 +167,11 @@ if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) fpsimd_save_state(¤t->thread.fpsimd_state); + if (fpsimd_settings && __this_cpu_read(fpsimd_stg_enable)) { + fpsimd_settings_disable(); + this_cpu_write(fpsimd_stg_enable, 0); + } + if (next->mm) { /* * If we are switching to a task whose most recent userland @@ -152,6 +189,14 @@ else set_ti_thread_flag(task_thread_info(next), TIF_FOREIGN_FPSTATE); + + if (!fpsimd_settings) + return; + + if (test_ti_thread_flag(task_thread_info(next), TIF_32BIT)) + fpsimd_enable_trap(); + else + fpsimd_disable_trap(); } } @@ -291,7 +336,7 @@ .notifier_call = fpsimd_cpu_pm_notifier, }; -static void fpsimd_pm_init(void) +static void __init fpsimd_pm_init(void) { cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block); } diff -ruw linux-4.4.115/arch/arm64/kernel/head.S linux-4.4.115-fbx/arch/arm64/kernel/head.S --- linux-4.4.115/arch/arm64/kernel/head.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/head.S 2019-10-29 09:26:23.001196895 +0100 @@ -25,10 +25,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -48,9 +50,6 @@ #error TEXT_OFFSET must be less than 2MB #endif -#define KERNEL_START _text -#define KERNEL_END _end - /* * Kernel startup entry point. * --------------------------- @@ -67,12 +66,11 @@ * in the entry routines. */ __HEAD - +_head: /* * DO NOT MODIFY. Image header expected by Linux boot-loaders. */ #ifdef CONFIG_EFI -efi_head: /* * This add instruction has no meaningful effect except that * its opcode forms the magic "MZ" signature required by UEFI. @@ -83,9 +81,9 @@ b stext // branch to kernel start, magic .long 0 // reserved #endif - .quad _kernel_offset_le // Image load offset from start of RAM, little-endian - .quad _kernel_size_le // Effective size of kernel image, little-endian - .quad _kernel_flags_le // Informative flags, little-endian + le64sym _kernel_offset_le // Image load offset from start of RAM, little-endian + le64sym _kernel_size_le // Effective size of kernel image, little-endian + le64sym _kernel_flags_le // Informative flags, little-endian .quad 0 // reserved .quad 0 // reserved .quad 0 // reserved @@ -94,14 +92,12 @@ .byte 0x4d .byte 0x64 #ifdef CONFIG_EFI - .long pe_header - efi_head // Offset to the PE header. + .long pe_header - _head // Offset to the PE header. #else .word 0 // reserved #endif #ifdef CONFIG_EFI - .globl __efistub_stext_offset - .set __efistub_stext_offset, stext - efi_head .align 3 pe_header: .ascii "PE" @@ -121,11 +117,11 @@ .short 0x20b // PE32+ format .byte 0x02 // MajorLinkerVersion .byte 0x14 // MinorLinkerVersion - .long _end - stext // SizeOfCode + .long _end - efi_header_end // SizeOfCode .long 0 // SizeOfInitializedData .long 0 // SizeOfUninitializedData - .long __efistub_entry - efi_head // AddressOfEntryPoint - .long __efistub_stext_offset // BaseOfCode + .long __efistub_entry - _head // AddressOfEntryPoint + .long efi_header_end - _head // BaseOfCode extra_header_fields: .quad 0 // ImageBase @@ -139,10 +135,10 @@ .short 0 // MinorSubsystemVersion .long 0 // Win32VersionValue - .long _end - efi_head // SizeOfImage + .long _end - _head // SizeOfImage // Everything before the kernel image is considered part of the header - .long __efistub_stext_offset // SizeOfHeaders + .long efi_header_end - _head // SizeOfHeaders .long 0 // CheckSum .short 0xa // Subsystem (EFI application) .short 0 // DllCharacteristics @@ -186,10 +182,10 @@ .byte 0 .byte 0 .byte 0 // end of 0 padding of section name - .long _end - stext // VirtualSize - .long __efistub_stext_offset // VirtualAddress - .long _edata - stext // SizeOfRawData - .long __efistub_stext_offset // PointerToRawData + .long _end - efi_header_end // VirtualSize + .long efi_header_end - _head // VirtualAddress + .long _edata - efi_header_end // SizeOfRawData + .long efi_header_end - _head // PointerToRawData .long 0 // PointerToRelocations (0 for executables) .long 0 // PointerToLineNumbers (0 for executables) @@ -198,19 +194,23 @@ .long 0xe0500020 // Characteristics (section flags) /* - * EFI will load stext onwards at the 4k section alignment + * EFI will load .text onwards at the 4k section alignment * described in the PE/COFF header. To ensure that instruction * sequences using an adrp and a :lo12: immediate will function - * correctly at this alignment, we must ensure that stext is + * correctly at this alignment, we must ensure that .text is * placed at a 4k boundary in the Image to begin with. */ .align 12 +efi_header_end: #endif + __INIT + ENTRY(stext) bl preserve_boot_args bl el2_setup // Drop to EL1, w20=cpu_boot_mode adrp x24, __PHYS_OFFSET + and x23, x24, MIN_KIMG_ALIGN - 1 // KASLR offset, defaults to 0 bl set_cpu_boot_mode_flag bl __create_page_tables // x25=TTBR0, x26=TTBR1 /* @@ -219,10 +219,10 @@ * On return, the CPU will be ready for the MMU to be turned on and * the TCR will have been set. */ - ldr x27, =__mmap_switched // address to jump to after + bl __cpu_setup // initialise processor + adr_l x27, __primary_switch // address to jump to after // MMU has been enabled - adr_l lr, __enable_mmu // return (PIC) address - b __cpu_setup // initialise processor + b __enable_mmu ENDPROC(stext) /* @@ -311,21 +311,21 @@ __create_page_tables: adrp x25, idmap_pg_dir adrp x26, swapper_pg_dir - mov x27, lr + mov x28, lr /* * Invalidate the idmap and swapper page tables to avoid potential * dirty cache lines being evicted. */ mov x0, x25 - add x1, x26, #SWAPPER_DIR_SIZE + add x1, x26, #SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE bl __inval_cache_range /* * Clear the idmap and swapper page tables. */ mov x0, x25 - add x6, x26, #SWAPPER_DIR_SIZE + add x6, x26, #SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE 1: stp xzr, xzr, [x0], #16 stp xzr, xzr, [x0], #16 stp xzr, xzr, [x0], #16 @@ -333,7 +333,7 @@ cmp x0, x6 b.lo 1b - ldr x7, =SWAPPER_MM_MMUFLAGS + mov x7, SWAPPER_MM_MMUFLAGS /* * Create the identity mapping. @@ -389,10 +389,13 @@ * Map the kernel image (starting with PHYS_OFFSET). */ mov x0, x26 // swapper_pg_dir - mov x5, #PAGE_OFFSET + mov_q x5, KIMAGE_VADDR + TEXT_OFFSET // compile time __va(_text) + add x5, x5, x23 // add KASLR displacement create_pgd_entry x0, x5, x3, x6 - ldr x6, =KERNEL_END // __va(KERNEL_END) - mov x3, x24 // phys offset + adrp x6, _end // runtime __pa(_end) + adrp x3, _text // runtime __pa(_text) + sub x6, x6, x3 // _end - _text + add x6, x6, x5 // runtime __va(_end) create_block_map x0, x7, x3, x5, x6 /* @@ -401,12 +404,11 @@ * tables again to remove any speculatively loaded cache lines. */ mov x0, x25 - add x1, x26, #SWAPPER_DIR_SIZE + add x1, x26, #SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE dmb sy bl __inval_cache_range - mov lr, x27 - ret + ret x28 ENDPROC(__create_page_tables) .ltorg @@ -414,30 +416,67 @@ * The following fragment of code is executed with the MMU enabled. */ .set initial_sp, init_thread_union + THREAD_START_SP -__mmap_switched: - adr_l x6, __bss_start - adr_l x7, __bss_stop - -1: cmp x6, x7 - b.hs 2f - str xzr, [x6], #8 // Clear BSS - b 1b -2: +__primary_switched: + mov x28, lr // preserve LR + + adr_l x8, vectors // load VBAR_EL1 with virtual + msr vbar_el1, x8 // vector table address + isb + + // Clear BSS + adr_l x0, __bss_start + mov x1, xzr + adr_l x2, __bss_stop + sub x2, x2, x0 + bl __pi_memset + dsb ishst // Make zero page visible to PTW + +#ifdef CONFIG_THREAD_INFO_IN_TASK + adrp x4, init_thread_union + add sp, x4, #THREAD_SIZE + adr_l x5, init_task + msr sp_el0, x5 // Save thread_info +#else adr_l sp, initial_sp, x4 + mov x4, sp + and x4, x4, #~(THREAD_SIZE - 1) + msr sp_el0, x4 // Save thread_info +#endif + str_l x21, __fdt_pointer, x5 // Save FDT pointer - str_l x24, memstart_addr, x6 // Save PHYS_OFFSET + + ldr_l x4, kimage_vaddr // Save the offset between + sub x4, x4, x24 // the kernel virtual and + str_l x4, kimage_voffset, x5 // physical mappings + mov x29, #0 #ifdef CONFIG_KASAN bl kasan_early_init #endif +#ifdef CONFIG_RANDOMIZE_BASE + tst x23, ~(MIN_KIMG_ALIGN - 1) // already running randomized? + b.ne 0f + mov x0, x21 // pass FDT address in x0 + mov x1, x23 // pass modulo offset in x1 + bl kaslr_early_init // parse FDT for KASLR options + cbz x0, 0f // KASLR disabled? just proceed + orr x23, x23, x0 // record KASLR offset + ret x28 // we must enable KASLR, return + // to __enable_mmu() +0: +#endif b start_kernel -ENDPROC(__mmap_switched) +ENDPROC(__primary_switched) /* * end early head section, begin head code that is also used for * hotplug and needs to have the same protections as the text region */ .section ".text","ax" + +ENTRY(kimage_vaddr) + .quad _text - TEXT_OFFSET + /* * If we're fortunate enough to boot at EL2, ensure that the world is * sane before dropping to EL1. @@ -544,7 +583,7 @@ * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed * in x20. See arch/arm64/include/asm/virt.h for more info. */ -ENTRY(set_cpu_boot_mode_flag) +set_cpu_boot_mode_flag: adr_l x1, __boot_cpu_mode cmp w20, #BOOT_CPU_MODE_EL2 b.ne 1f @@ -577,7 +616,7 @@ bl el2_setup // Drop to EL1, w20=cpu_boot_mode bl set_cpu_boot_mode_flag mrs x0, mpidr_el1 - ldr x1, =MPIDR_HWID_BITMASK + mov_q x1, MPIDR_HWID_BITMASK and x0, x0, x1 adr_l x3, secondary_holding_pen_release pen: ldr x4, [x3] @@ -597,7 +636,7 @@ b secondary_startup ENDPROC(secondary_entry) -ENTRY(secondary_startup) +secondary_startup: /* * Common entry point for secondary CPUs. */ @@ -605,14 +644,26 @@ adrp x26, swapper_pg_dir bl __cpu_setup // initialise processor - ldr x21, =secondary_data - ldr x27, =__secondary_switched // address to jump to after enabling the MMU + adr_l x27, __secondary_switch // address to jump to after enabling the MMU b __enable_mmu ENDPROC(secondary_startup) -ENTRY(__secondary_switched) - ldr x0, [x21] // get secondary_data.stack +__secondary_switched: + adr_l x5, vectors + msr vbar_el1, x5 + isb +#ifdef CONFIG_THREAD_INFO_IN_TASK + adr_l x0, secondary_data + ldr x1, [x0, #CPU_BOOT_STACK] // get secondary_data.stack + mov sp, x1 + ldr x2, [x0, #CPU_BOOT_TASK] + msr sp_el0, x2 +#else + ldr_l x0, secondary_data // get secondary_data.stack mov sp, x0 + and x0, x0, #~(THREAD_SIZE - 1) + msr sp_el0, x0 // save thread_info +#endif mov x29, #0 b secondary_start_kernel ENDPROC(__secondary_switched) @@ -629,13 +680,12 @@ * If it isn't, park the CPU */ .section ".idmap.text", "ax" -__enable_mmu: +ENTRY(__enable_mmu) + mrs x22, sctlr_el1 // preserve old SCTLR_EL1 value mrs x1, ID_AA64MMFR0_EL1 ubfx x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4 cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED b.ne __no_granule_support - ldr x5, =vectors - msr vbar_el1, x5 msr ttbr0_el1, x25 // load TTBR0 msr ttbr1_el1, x26 // load TTBR1 isb @@ -649,6 +699,28 @@ ic iallu dsb nsh isb +#ifdef CONFIG_RANDOMIZE_BASE + mov x19, x0 // preserve new SCTLR_EL1 value + blr x27 + + /* + * If we return here, we have a KASLR displacement in x23 which we need + * to take into account by discarding the current kernel mapping and + * creating a new one. + */ + msr sctlr_el1, x22 // disable the MMU + isb + bl __create_page_tables // recreate kernel mapping + + tlbi vmalle1 // Remove any stale TLB entries + dsb nsh + + msr sctlr_el1, x19 // re-enable the MMU + isb + ic iallu // flush instructions fetched + dsb nsh // via old mapping + isb +#endif br x27 ENDPROC(__enable_mmu) @@ -656,3 +728,38 @@ wfe b __no_granule_support ENDPROC(__no_granule_support) + +__primary_switch: +#ifdef CONFIG_RELOCATABLE + /* + * Iterate over each entry in the relocation table, and apply the + * relocations in place. + */ + ldr w9, =__rela_offset // offset to reloc table + ldr w10, =__rela_size // size of reloc table + + mov_q x11, KIMAGE_VADDR // default virtual offset + add x11, x11, x23 // actual virtual offset + add x9, x9, x11 // __va(.rela) + add x10, x9, x10 // __va(.rela) + sizeof(.rela) + +0: cmp x9, x10 + b.hs 1f + ldp x11, x12, [x9], #24 + ldr x13, [x9, #-8] + cmp w12, #R_AARCH64_RELATIVE + b.ne 0b + add x13, x13, x23 // relocate + str x13, [x11, x23] + b 0b + +1: +#endif + ldr x8, =__primary_switched + br x8 +ENDPROC(__primary_switch) + +__secondary_switch: + ldr x8, =__secondary_switched + br x8 +ENDPROC(__secondary_switch) diff -ruw linux-4.4.115/arch/arm64/kernel/hyp-stub.S linux-4.4.115-fbx/arch/arm64/kernel/hyp-stub.S --- linux-4.4.115/arch/arm64/kernel/hyp-stub.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/hyp-stub.S 2019-10-29 09:26:23.005196935 +0100 @@ -22,6 +22,8 @@ #include #include +#include +#include #include #include @@ -53,15 +55,26 @@ .align 11 el1_sync: - mrs x1, esr_el2 - lsr x1, x1, #26 - cmp x1, #0x16 - b.ne 2f // Not an HVC trap - cbz x0, 1f - msr vbar_el2, x0 // Set vbar_el2 - b 2f -1: mrs x0, vbar_el2 // Return vbar_el2 -2: eret + mrs x30, esr_el2 + lsr x30, x30, #ESR_ELx_EC_SHIFT + + cmp x30, #ESR_ELx_EC_HVC64 + b.ne 9f // Not an HVC trap + + cmp x0, #HVC_GET_VECTORS + b.ne 1f + mrs x0, vbar_el2 + b 9f + +1: cmp x0, #HVC_SET_VECTORS + b.ne 2f + msr vbar_el2, x1 + b 9f + + /* Someone called kvm_call_hyp() against the hyp-stub... */ +2: mov x0, #ARM_EXCEPTION_HYP_GONE + +9: eret ENDPROC(el1_sync) .macro invalid_vector label @@ -101,10 +114,18 @@ */ ENTRY(__hyp_get_vectors) - mov x0, xzr - // fall through -ENTRY(__hyp_set_vectors) + str lr, [sp, #-16]! + mov x0, #HVC_GET_VECTORS hvc #0 + ldr lr, [sp], #16 ret ENDPROC(__hyp_get_vectors) + +ENTRY(__hyp_set_vectors) + str lr, [sp, #-16]! + mov x1, x0 + mov x0, #HVC_SET_VECTORS + hvc #0 + ldr lr, [sp], #16 + ret ENDPROC(__hyp_set_vectors) diff -ruw linux-4.4.115/arch/arm64/kernel/image.h linux-4.4.115-fbx/arch/arm64/kernel/image.h --- linux-4.4.115/arch/arm64/kernel/image.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/image.h 2019-10-29 09:26:23.005196935 +0100 @@ -26,21 +26,27 @@ * There aren't any ELF relocations we can use to endian-swap values known only * at link time (e.g. the subtraction of two symbol addresses), so we must get * the linker to endian-swap certain values before emitting them. + * + * Note that, in order for this to work when building the ELF64 PIE executable + * (for KASLR), these values should not be referenced via R_AARCH64_ABS64 + * relocations, since these are fixed up at runtime rather than at build time + * when PIE is in effect. So we need to split them up in 32-bit high and low + * words. */ #ifdef CONFIG_CPU_BIG_ENDIAN -#define DATA_LE64(data) \ - ((((data) & 0x00000000000000ff) << 56) | \ - (((data) & 0x000000000000ff00) << 40) | \ - (((data) & 0x0000000000ff0000) << 24) | \ - (((data) & 0x00000000ff000000) << 8) | \ - (((data) & 0x000000ff00000000) >> 8) | \ - (((data) & 0x0000ff0000000000) >> 24) | \ - (((data) & 0x00ff000000000000) >> 40) | \ - (((data) & 0xff00000000000000) >> 56)) +#define DATA_LE32(data) \ + ((((data) & 0x000000ff) << 24) | \ + (((data) & 0x0000ff00) << 8) | \ + (((data) & 0x00ff0000) >> 8) | \ + (((data) & 0xff000000) >> 24)) #else -#define DATA_LE64(data) ((data) & 0xffffffffffffffff) +#define DATA_LE32(data) ((data) & 0xffffffff) #endif +#define DEFINE_IMAGE_LE64(sym, data) \ + sym##_lo32 = DATA_LE32((data) & 0xffffffff); \ + sym##_hi32 = DATA_LE32((data) >> 32) + #ifdef CONFIG_CPU_BIG_ENDIAN #define __HEAD_FLAG_BE 1 #else @@ -49,8 +55,11 @@ #define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2) +#define __HEAD_FLAG_PHYS_BASE 1 + #define __HEAD_FLAGS ((__HEAD_FLAG_BE << 0) | \ - (__HEAD_FLAG_PAGE_SIZE << 1)) + (__HEAD_FLAG_PAGE_SIZE << 1) | \ + (__HEAD_FLAG_PHYS_BASE << 3)) /* * These will output as part of the Image header, which should be little-endian @@ -58,12 +67,24 @@ * endian swapped in head.S, all are done here for consistency. */ #define HEAD_SYMBOLS \ - _kernel_size_le = DATA_LE64(_end - _text); \ - _kernel_offset_le = DATA_LE64(TEXT_OFFSET); \ - _kernel_flags_le = DATA_LE64(__HEAD_FLAGS); + DEFINE_IMAGE_LE64(_kernel_size_le, _end - _text); \ + DEFINE_IMAGE_LE64(_kernel_offset_le, TEXT_OFFSET); \ + DEFINE_IMAGE_LE64(_kernel_flags_le, __HEAD_FLAGS); #ifdef CONFIG_EFI +__efistub_stext_offset = stext - _text; + +/* + * Prevent the symbol aliases below from being emitted into the kallsyms + * table, by forcing them to be absolute symbols (which are conveniently + * ignored by scripts/kallsyms) rather than section relative symbols. + * The distinction is only relevant for partial linking, and only for symbols + * that are defined within a section declaration (which is not the case for + * the definitions below) so the resulting values will be identical. + */ +#define KALLSYMS_HIDE(sym) ABSOLUTE(sym) + /* * The EFI stub has its own symbol namespace prefixed by __efistub_, to * isolate it from the kernel proper. The following symbols are legally @@ -73,25 +94,25 @@ * linked at. The routines below are all implemented in assembler in a * position independent manner */ -__efistub_memcmp = __pi_memcmp; -__efistub_memchr = __pi_memchr; -__efistub_memcpy = __pi_memcpy; -__efistub_memmove = __pi_memmove; -__efistub_memset = __pi_memset; -__efistub_strlen = __pi_strlen; -__efistub_strcmp = __pi_strcmp; -__efistub_strncmp = __pi_strncmp; -__efistub___flush_dcache_area = __pi___flush_dcache_area; +__efistub_memcmp = KALLSYMS_HIDE(__pi_memcmp); +__efistub_memchr = KALLSYMS_HIDE(__pi_memchr); +__efistub_memcpy = KALLSYMS_HIDE(__pi_memcpy); +__efistub_memmove = KALLSYMS_HIDE(__pi_memmove); +__efistub_memset = KALLSYMS_HIDE(__pi_memset); +__efistub_strlen = KALLSYMS_HIDE(__pi_strlen); +__efistub_strcmp = KALLSYMS_HIDE(__pi_strcmp); +__efistub_strncmp = KALLSYMS_HIDE(__pi_strncmp); +__efistub___flush_dcache_area = KALLSYMS_HIDE(__pi___flush_dcache_area); #ifdef CONFIG_KASAN -__efistub___memcpy = __pi_memcpy; -__efistub___memmove = __pi_memmove; -__efistub___memset = __pi_memset; +__efistub___memcpy = KALLSYMS_HIDE(__pi_memcpy); +__efistub___memmove = KALLSYMS_HIDE(__pi_memmove); +__efistub___memset = KALLSYMS_HIDE(__pi_memset); #endif -__efistub__text = _text; -__efistub__end = _end; -__efistub__edata = _edata; +__efistub__text = KALLSYMS_HIDE(_text); +__efistub__end = KALLSYMS_HIDE(_end); +__efistub__edata = KALLSYMS_HIDE(_edata); #endif diff -ruw linux-4.4.115/arch/arm64/kernel/insn.c linux-4.4.115-fbx/arch/arm64/kernel/insn.c --- linux-4.4.115/arch/arm64/kernel/insn.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/insn.c 2019-01-22 16:16:21.551228730 +0100 @@ -2,7 +2,7 @@ * Copyright (C) 2013 Huawei Ltd. * Author: Jiang Liu * - * Copyright (C) 2014 Zi Shen Lim + * Copyright (C) 2014-2016 Zi Shen Lim * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -30,6 +30,7 @@ #include #include #include +#include #include #define AARCH64_INSN_SF_BIT BIT(31) @@ -95,8 +96,9 @@ if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX)) page = vmalloc_to_page(addr); - else if (!module && IS_ENABLED(CONFIG_DEBUG_RODATA)) - page = virt_to_page(addr); + else if (!module && (IS_ENABLED(CONFIG_DEBUG_RODATA) + || IS_ENABLED(CONFIG_KERNEL_TEXT_RDONLY))) + page = phys_to_page(__pa_symbol(addr)); else return addr; @@ -162,6 +164,32 @@ aarch64_insn_is_nop(insn); } +bool __kprobes aarch64_insn_uses_literal(u32 insn) +{ + /* ldr/ldrsw (literal), prfm */ + + return aarch64_insn_is_ldr_lit(insn) || + aarch64_insn_is_ldrsw_lit(insn) || + aarch64_insn_is_adr_adrp(insn) || + aarch64_insn_is_prfm_lit(insn); +} + +bool __kprobes aarch64_insn_is_branch(u32 insn) +{ + /* b, bl, cb*, tb*, b.cond, br, blr */ + + return aarch64_insn_is_b(insn) || + aarch64_insn_is_bl(insn) || + aarch64_insn_is_cbz(insn) || + aarch64_insn_is_cbnz(insn) || + aarch64_insn_is_tbz(insn) || + aarch64_insn_is_tbnz(insn) || + aarch64_insn_is_ret(insn) || + aarch64_insn_is_br(insn) || + aarch64_insn_is_blr(insn) || + aarch64_insn_is_bcond(insn); +} + /* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section B2.6.5 "Concurrent modification and execution of instructions": @@ -363,6 +391,9 @@ u32 immlo, immhi, mask; int shift; + if (insn == AARCH64_BREAK_FAULT) + return AARCH64_BREAK_FAULT; + switch (type) { case AARCH64_INSN_IMM_ADR: shift = 0; @@ -377,7 +408,7 @@ if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", type); - return 0; + return AARCH64_BREAK_FAULT; } } @@ -394,9 +425,12 @@ { int shift; + if (insn == AARCH64_BREAK_FAULT) + return AARCH64_BREAK_FAULT; + if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) { pr_err("%s: unknown register encoding %d\n", __func__, reg); - return 0; + return AARCH64_BREAK_FAULT; } switch (type) { @@ -417,7 +451,7 @@ default: pr_err("%s: unknown register type encoding %d\n", __func__, type); - return 0; + return AARCH64_BREAK_FAULT; } insn &= ~(GENMASK(4, 0) << shift); @@ -446,7 +480,7 @@ break; default: pr_err("%s: unknown size encoding %d\n", __func__, type); - return 0; + return AARCH64_BREAK_FAULT; } insn &= ~GENMASK(31, 30); @@ -460,14 +494,17 @@ { long offset; - /* - * PC: A 64-bit Program Counter holding the address of the current - * instruction. A64 instructions must be word-aligned. - */ - BUG_ON((pc & 0x3) || (addr & 0x3)); + if ((pc & 0x3) || (addr & 0x3)) { + pr_err("%s: A64 instructions must be word aligned\n", __func__); + return range; + } offset = ((long)addr - (long)pc); - BUG_ON(offset < -range || offset >= range); + + if (offset < -range || offset >= range) { + pr_err("%s: offset out of range\n", __func__); + return range; + } return offset; } @@ -484,6 +521,8 @@ * texts are within +/-128M. */ offset = branch_imm_common(pc, addr, SZ_128M); + if (offset >= SZ_128M) + return AARCH64_BREAK_FAULT; switch (type) { case AARCH64_INSN_BRANCH_LINK: @@ -493,7 +532,7 @@ insn = aarch64_insn_get_b_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown branch encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -510,6 +549,8 @@ long offset; offset = branch_imm_common(pc, addr, SZ_1M); + if (offset >= SZ_1M) + return AARCH64_BREAK_FAULT; switch (type) { case AARCH64_INSN_BRANCH_COMP_ZERO: @@ -519,7 +560,7 @@ insn = aarch64_insn_get_cbnz_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown branch encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -530,7 +571,7 @@ insn |= AARCH64_INSN_SF_BIT; break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -550,7 +591,10 @@ insn = aarch64_insn_get_bcond_value(); - BUG_ON(cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL); + if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) { + pr_err("%s: unknown condition encoding %d\n", __func__, cond); + return AARCH64_BREAK_FAULT; + } insn |= cond; return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, @@ -583,7 +627,7 @@ insn = aarch64_insn_get_ret_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown branch encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -606,7 +650,7 @@ insn = aarch64_insn_get_str_reg_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown load/store encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -645,26 +689,30 @@ insn = aarch64_insn_get_stp_post_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown load/store encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } switch (variant) { case AARCH64_INSN_VARIANT_32BIT: - /* offset must be multiples of 4 in the range [-256, 252] */ - BUG_ON(offset & 0x3); - BUG_ON(offset < -256 || offset > 252); + if ((offset & 0x3) || (offset < -256) || (offset > 252)) { + pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n", + __func__, offset); + return AARCH64_BREAK_FAULT; + } shift = 2; break; case AARCH64_INSN_VARIANT_64BIT: - /* offset must be multiples of 8 in the range [-512, 504] */ - BUG_ON(offset & 0x7); - BUG_ON(offset < -512 || offset > 504); + if ((offset & 0x7) || (offset < -512) || (offset > 504)) { + pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n", + __func__, offset); + return AARCH64_BREAK_FAULT; + } shift = 3; insn |= AARCH64_INSN_SF_BIT; break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -702,7 +750,7 @@ insn = aarch64_insn_get_subs_imm_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown add/sub encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -713,11 +761,14 @@ insn |= AARCH64_INSN_SF_BIT; break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } - BUG_ON(imm & ~(SZ_4K - 1)); + if (imm & ~(SZ_4K - 1)) { + pr_err("%s: invalid immediate encoding %d\n", __func__, imm); + return AARCH64_BREAK_FAULT; + } insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); @@ -746,7 +797,7 @@ insn = aarch64_insn_get_sbfm_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown bitfield encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -759,12 +810,18 @@ mask = GENMASK(5, 0); break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } - BUG_ON(immr & ~mask); - BUG_ON(imms & ~mask); + if (immr & ~mask) { + pr_err("%s: invalid immr encoding %d\n", __func__, immr); + return AARCH64_BREAK_FAULT; + } + if (imms & ~mask) { + pr_err("%s: invalid imms encoding %d\n", __func__, imms); + return AARCH64_BREAK_FAULT; + } insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); @@ -793,23 +850,33 @@ insn = aarch64_insn_get_movn_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown movewide encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } - BUG_ON(imm & ~(SZ_64K - 1)); + if (imm & ~(SZ_64K - 1)) { + pr_err("%s: invalid immediate encoding %d\n", __func__, imm); + return AARCH64_BREAK_FAULT; + } switch (variant) { case AARCH64_INSN_VARIANT_32BIT: - BUG_ON(shift != 0 && shift != 16); + if (shift != 0 && shift != 16) { + pr_err("%s: invalid shift encoding %d\n", __func__, + shift); + return AARCH64_BREAK_FAULT; + } break; case AARCH64_INSN_VARIANT_64BIT: insn |= AARCH64_INSN_SF_BIT; - BUG_ON(shift != 0 && shift != 16 && shift != 32 && - shift != 48); + if (shift != 0 && shift != 16 && shift != 32 && shift != 48) { + pr_err("%s: invalid shift encoding %d\n", __func__, + shift); + return AARCH64_BREAK_FAULT; + } break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -843,20 +910,28 @@ insn = aarch64_insn_get_subs_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown add/sub encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } switch (variant) { case AARCH64_INSN_VARIANT_32BIT: - BUG_ON(shift & ~(SZ_32 - 1)); + if (shift & ~(SZ_32 - 1)) { + pr_err("%s: invalid shift encoding %d\n", __func__, + shift); + return AARCH64_BREAK_FAULT; + } break; case AARCH64_INSN_VARIANT_64BIT: insn |= AARCH64_INSN_SF_BIT; - BUG_ON(shift & ~(SZ_64 - 1)); + if (shift & ~(SZ_64 - 1)) { + pr_err("%s: invalid shift encoding %d\n", __func__, + shift); + return AARCH64_BREAK_FAULT; + } break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -885,11 +960,15 @@ insn = aarch64_insn_get_rev32_value(); break; case AARCH64_INSN_DATA1_REVERSE_64: - BUG_ON(variant != AARCH64_INSN_VARIANT_64BIT); + if (variant != AARCH64_INSN_VARIANT_64BIT) { + pr_err("%s: invalid variant for reverse64 %d\n", + __func__, variant); + return AARCH64_BREAK_FAULT; + } insn = aarch64_insn_get_rev64_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown data1 encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -900,7 +979,7 @@ insn |= AARCH64_INSN_SF_BIT; break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -937,7 +1016,7 @@ insn = aarch64_insn_get_rorv_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown data2 encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -948,7 +1027,7 @@ insn |= AARCH64_INSN_SF_BIT; break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -976,7 +1055,7 @@ insn = aarch64_insn_get_msub_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown data3 encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } @@ -987,7 +1066,7 @@ insn |= AARCH64_INSN_SF_BIT; break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -1037,20 +1116,28 @@ insn = aarch64_insn_get_bics_value(); break; default: - BUG_ON(1); + pr_err("%s: unknown logical encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; } switch (variant) { case AARCH64_INSN_VARIANT_32BIT: - BUG_ON(shift & ~(SZ_32 - 1)); + if (shift & ~(SZ_32 - 1)) { + pr_err("%s: invalid shift encoding %d\n", __func__, + shift); + return AARCH64_BREAK_FAULT; + } break; case AARCH64_INSN_VARIANT_64BIT: insn |= AARCH64_INSN_SF_BIT; - BUG_ON(shift & ~(SZ_64 - 1)); + if (shift & ~(SZ_64 - 1)) { + pr_err("%s: invalid shift encoding %d\n", __func__, + shift); + return AARCH64_BREAK_FAULT; + } break; default: - BUG_ON(1); + pr_err("%s: unknown variant encoding %d\n", __func__, variant); return AARCH64_BREAK_FAULT; } @@ -1116,6 +1203,14 @@ BUG(); } +/* + * Extract the Op/CR data from a msr/mrs instruction. + */ +u32 aarch64_insn_extract_system_reg(u32 insn) +{ + return (insn & 0x1FFFE0) >> 5; +} + bool aarch32_insn_is_wide(u32 insn) { return insn >= 0xe800; @@ -1141,3 +1236,101 @@ { return insn & CRM_MASK; } + +static bool __kprobes __check_eq(unsigned long pstate) +{ + return (pstate & PSR_Z_BIT) != 0; +} + +static bool __kprobes __check_ne(unsigned long pstate) +{ + return (pstate & PSR_Z_BIT) == 0; +} + +static bool __kprobes __check_cs(unsigned long pstate) +{ + return (pstate & PSR_C_BIT) != 0; +} + +static bool __kprobes __check_cc(unsigned long pstate) +{ + return (pstate & PSR_C_BIT) == 0; +} + +static bool __kprobes __check_mi(unsigned long pstate) +{ + return (pstate & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_pl(unsigned long pstate) +{ + return (pstate & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_vs(unsigned long pstate) +{ + return (pstate & PSR_V_BIT) != 0; +} + +static bool __kprobes __check_vc(unsigned long pstate) +{ + return (pstate & PSR_V_BIT) == 0; +} + +static bool __kprobes __check_hi(unsigned long pstate) +{ + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (pstate & PSR_C_BIT) != 0; +} + +static bool __kprobes __check_ls(unsigned long pstate) +{ + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (pstate & PSR_C_BIT) == 0; +} + +static bool __kprobes __check_ge(unsigned long pstate) +{ + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (pstate & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_lt(unsigned long pstate) +{ + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (pstate & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_gt(unsigned long pstate) +{ + /*PSR_N_BIT ^= PSR_V_BIT */ + unsigned long temp = pstate ^ (pstate << 3); + + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ + return (temp & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_le(unsigned long pstate) +{ + /*PSR_N_BIT ^= PSR_V_BIT */ + unsigned long temp = pstate ^ (pstate << 3); + + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ + return (temp & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_al(unsigned long pstate) +{ + return true; +} + +/* + * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that + * it behaves identically to 0b1110 ("al"). + */ +pstate_check_t * const aarch32_opcode_cond_checks[16] = { + __check_eq, __check_ne, __check_cs, __check_cc, + __check_mi, __check_pl, __check_vs, __check_vc, + __check_hi, __check_ls, __check_ge, __check_lt, + __check_gt, __check_le, __check_al, __check_al +}; diff -ruw linux-4.4.115/arch/arm64/kernel/io.c linux-4.4.115-fbx/arch/arm64/kernel/io.c --- linux-4.4.115/arch/arm64/kernel/io.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/io.c 2019-01-22 16:16:21.551228730 +0100 @@ -19,29 +19,29 @@ #include #include #include +#include /* * Copy data from IO memory space to "real" memory space. */ void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count) { - while (count && (!IS_ALIGNED((unsigned long)from, 8) || - !IS_ALIGNED((unsigned long)to, 8))) { - *(u8 *)to = __raw_readb(from); + while (count && !IS_ALIGNED((unsigned long)from, 8)) { + *(u8 *)to = __raw_readb_no_log(from); from++; to++; count--; } while (count >= 8) { - *(u64 *)to = __raw_readq(from); + *(u64 *)to = __raw_readq_no_log(from); from += 8; to += 8; count -= 8; } while (count) { - *(u8 *)to = __raw_readb(from); + *(u8 *)to = __raw_readb_no_log(from); from++; to++; count--; @@ -54,23 +54,22 @@ */ void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count) { - while (count && (!IS_ALIGNED((unsigned long)to, 8) || - !IS_ALIGNED((unsigned long)from, 8))) { - __raw_writeb(*(volatile u8 *)from, to); + while (count && !IS_ALIGNED((unsigned long)to, 8)) { + __raw_writeb_no_log(*(u8 *)from, to); from++; to++; count--; } while (count >= 8) { - __raw_writeq(*(volatile u64 *)from, to); + __raw_writeq_no_log(*(u64 *)from, to); from += 8; to += 8; count -= 8; } while (count) { - __raw_writeb(*(volatile u8 *)from, to); + __raw_writeb_no_log(*(u8 *)from, to); from++; to++; count--; @@ -90,19 +89,19 @@ qc |= qc << 32; while (count && !IS_ALIGNED((unsigned long)dst, 8)) { - __raw_writeb(c, dst); + __raw_writeb_no_log(c, dst); dst++; count--; } while (count >= 8) { - __raw_writeq(qc, dst); + __raw_writeq_no_log(qc, dst); dst += 8; count -= 8; } while (count) { - __raw_writeb(c, dst); + __raw_writeb_no_log(c, dst); dst++; count--; } diff -ruw linux-4.4.115/arch/arm64/kernel/irq.c linux-4.4.115-fbx/arch/arm64/kernel/irq.c --- linux-4.4.115/arch/arm64/kernel/irq.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/irq.c 2019-01-22 16:16:21.551228730 +0100 @@ -30,6 +30,9 @@ unsigned long irq_err_count; +/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ +DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16); + int arch_show_interrupts(struct seq_file *p, int prec) { show_ipi_list(p, prec); diff -ruw linux-4.4.115/arch/arm64/kernel/Makefile linux-4.4.115-fbx/arch/arm64/kernel/Makefile --- linux-4.4.115/arch/arm64/kernel/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/Makefile 2019-01-22 16:16:21.547228694 +0100 @@ -14,10 +14,10 @@ arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \ + hyp-stub.o psci.o cpu_ops.o insn.o \ return_address.o cpuinfo.o cpu_errata.o \ cpufeature.o alternative.o cacheinfo.o \ - smp.o smp_spin_table.o topology.o + smp.o smp_spin_table.o topology.o smccc-call.o extra-$(CONFIG_EFI) := efi-entry.o @@ -26,12 +26,13 @@ $(call if_changed,objcopy) arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ - sys_compat.o entry32.o \ - ../../arm/kernel/opcodes.o + sys_compat.o entry32.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o +arm64-obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o -arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o +arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_debug.o perf_trace_counters.o \ + perf_trace_user.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o @@ -41,12 +42,18 @@ arm64-obj-$(CONFIG_PCI) += pci.o arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o arm64-obj-$(CONFIG_ACPI) += acpi.o +arm64-obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o +arm64-obj-$(CONFIG_MSM_APP_API) += app_api.o +arm64-obj-$(CONFIG_MSM_APP_SETTINGS) += app_setting.o +arm64-obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o +arm64-obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o +arm64-obj-$(CONFIG_PARAVIRT) += paravirt.o -obj-y += $(arm64-obj-y) vdso/ +ifeq ($(CONFIG_KVM),y) +arm64-obj-$(CONFIG_HARDEN_BRANCH_PREDICTOR) += bpi.o +endif + +obj-y += $(arm64-obj-y) vdso/ probes/ obj-m += $(arm64-obj-m) head-y := head.o extra-y += $(head-y) vmlinux.lds - -# vDSO - this must be built first to generate the symbol offsets -$(call objectify,$(arm64-obj-y)): $(obj)/vdso/vdso-offsets.h -$(obj)/vdso/vdso-offsets.h: $(obj)/vdso diff -ruw linux-4.4.115/arch/arm64/kernel/module.c linux-4.4.115-fbx/arch/arm64/kernel/module.c --- linux-4.4.115/arch/arm64/kernel/module.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/module.c 2019-10-29 09:26:23.005196935 +0100 @@ -30,17 +30,30 @@ #include #include -#define AARCH64_INSN_IMM_MOVNZ AARCH64_INSN_IMM_MAX -#define AARCH64_INSN_IMM_MOVK AARCH64_INSN_IMM_16 - void *module_alloc(unsigned long size) { void *p; - p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END, + p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, + module_alloc_base + MODULES_VSIZE, GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, __builtin_return_address(0)); + if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && + !IS_ENABLED(CONFIG_KASAN)) + /* + * KASAN can only deal with module allocations being served + * from the reserved module region, since the remainder of + * the vmalloc region is already backed by zero shadow pages, + * and punching holes into it is non-trivial. Since the module + * region is not randomized when KASAN is enabled, it is even + * less likely that the module region gets exhausted, so we + * can simply omit this fallback in that case. + */ + p = __vmalloc_node_range(size, MODULE_ALIGN, VMALLOC_START, + VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_EXEC, 0, + NUMA_NO_NODE, __builtin_return_address(0)); + if (p && (kasan_module_alloc(p, size) < 0)) { vfree(p); return NULL; @@ -75,15 +88,18 @@ static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len) { - u64 imm_mask = (1 << len) - 1; s64 sval = do_reloc(op, place, val); switch (len) { case 16: *(s16 *)place = sval; + if (sval < S16_MIN || sval > U16_MAX) + return -ERANGE; break; case 32: *(s32 *)place = sval; + if (sval < S32_MIN || sval > U32_MAX) + return -ERANGE; break; case 64: *(s64 *)place = sval; @@ -92,34 +108,23 @@ pr_err("Invalid length (%d) for data relocation\n", len); return 0; } - - /* - * Extract the upper value bits (including the sign bit) and - * shift them to bit 0. - */ - sval = (s64)(sval & ~(imm_mask >> 1)) >> (len - 1); - - /* - * Overflow has occurred if the value is not representable in - * len bits (i.e the bottom len bits are not sign-extended and - * the top bits are not all zero). - */ - if ((u64)(sval + 1) > 2) - return -ERANGE; - return 0; } +enum aarch64_insn_movw_imm_type { + AARCH64_INSN_IMM_MOVNZ, + AARCH64_INSN_IMM_MOVKZ, +}; + static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val, - int lsb, enum aarch64_insn_imm_type imm_type) + int lsb, enum aarch64_insn_movw_imm_type imm_type) { - u64 imm, limit = 0; + u64 imm; s64 sval; u32 insn = le32_to_cpu(*(u32 *)place); sval = do_reloc(op, place, val); - sval >>= lsb; - imm = sval & 0xffff; + imm = sval >> lsb; if (imm_type == AARCH64_INSN_IMM_MOVNZ) { /* @@ -128,7 +133,7 @@ * immediate is less than zero. */ insn &= ~(3 << 29); - if ((s64)imm >= 0) { + if (sval >= 0) { /* >=0: Set the instruction to MOVZ (opcode 10b). */ insn |= 2 << 29; } else { @@ -140,29 +145,13 @@ */ imm = ~imm; } - imm_type = AARCH64_INSN_IMM_MOVK; } /* Update the instruction with the new encoding. */ - insn = aarch64_insn_encode_immediate(imm_type, insn, imm); + insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); *(u32 *)place = cpu_to_le32(insn); - /* Shift out the immediate field. */ - sval >>= 16; - - /* - * For unsigned immediates, the overflow check is straightforward. - * For signed immediates, the sign bit is actually the bit past the - * most significant bit of the field. - * The AARCH64_INSN_IMM_16 immediate type is unsigned. - */ - if (imm_type != AARCH64_INSN_IMM_16) { - sval++; - limit++; - } - - /* Check the upper bits depending on the sign of the immediate. */ - if ((u64)sval > limit) + if (imm > U16_MAX) return -ERANGE; return 0; @@ -267,25 +256,25 @@ overflow_check = false; case R_AARCH64_MOVW_UABS_G0: ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, - AARCH64_INSN_IMM_16); + AARCH64_INSN_IMM_MOVKZ); break; case R_AARCH64_MOVW_UABS_G1_NC: overflow_check = false; case R_AARCH64_MOVW_UABS_G1: ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, - AARCH64_INSN_IMM_16); + AARCH64_INSN_IMM_MOVKZ); break; case R_AARCH64_MOVW_UABS_G2_NC: overflow_check = false; case R_AARCH64_MOVW_UABS_G2: ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, - AARCH64_INSN_IMM_16); + AARCH64_INSN_IMM_MOVKZ); break; case R_AARCH64_MOVW_UABS_G3: /* We're using the top bits so we can't overflow. */ overflow_check = false; ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48, - AARCH64_INSN_IMM_16); + AARCH64_INSN_IMM_MOVKZ); break; case R_AARCH64_MOVW_SABS_G0: ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, @@ -302,7 +291,7 @@ case R_AARCH64_MOVW_PREL_G0_NC: overflow_check = false; ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, - AARCH64_INSN_IMM_MOVK); + AARCH64_INSN_IMM_MOVKZ); break; case R_AARCH64_MOVW_PREL_G0: ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, @@ -311,7 +300,7 @@ case R_AARCH64_MOVW_PREL_G1_NC: overflow_check = false; ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, - AARCH64_INSN_IMM_MOVK); + AARCH64_INSN_IMM_MOVKZ); break; case R_AARCH64_MOVW_PREL_G1: ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, @@ -320,7 +309,7 @@ case R_AARCH64_MOVW_PREL_G2_NC: overflow_check = false; ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, - AARCH64_INSN_IMM_MOVK); + AARCH64_INSN_IMM_MOVKZ); break; case R_AARCH64_MOVW_PREL_G2: ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, @@ -388,6 +377,13 @@ case R_AARCH64_CALL26: ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, AARCH64_INSN_IMM_26); + + if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && + ovf == -ERANGE) { + val = module_emit_plt_entry(me, &rel[i], sym); + ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, + 26, AARCH64_INSN_IMM_26); + } break; default: diff -ruw linux-4.4.115/arch/arm64/kernel/perf_callchain.c linux-4.4.115-fbx/arch/arm64/kernel/perf_callchain.c --- linux-4.4.115/arch/arm64/kernel/perf_callchain.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/perf_callchain.c 2019-01-22 16:16:21.551228730 +0100 @@ -164,8 +164,11 @@ frame.fp = regs->regs[29]; frame.sp = regs->sp; frame.pc = regs->pc; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + frame.graph = current->curr_ret_stack; +#endif - walk_stackframe(&frame, callchain_trace, entry); + walk_stackframe(current, &frame, callchain_trace, entry); } unsigned long perf_instruction_pointer(struct pt_regs *regs) diff -ruw linux-4.4.115/arch/arm64/kernel/process.c linux-4.4.115-fbx/arch/arm64/kernel/process.c --- linux-4.4.115/arch/arm64/kernel/process.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/process.c 2019-10-29 09:26:23.005196935 +0100 @@ -45,9 +45,14 @@ #include #include #include +#ifdef CONFIG_THREAD_INFO_IN_TASK +#include +#endif +#include #include #include +#include #include #include #include @@ -82,6 +87,16 @@ trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id()); } +void arch_cpu_idle_enter(void) +{ + idle_notifier_call_chain(IDLE_START); +} + +void arch_cpu_idle_exit(void) +{ + idle_notifier_call_chain(IDLE_END); +} + #ifdef CONFIG_HOTPLUG_CPU void arch_cpu_idle_dead(void) { @@ -164,6 +179,64 @@ while (1); } +/* + * dump a block of kernel memory from around the given address + */ +static void show_data(unsigned long addr, int nbytes, const char *name) +{ + int i, j; + int nlines; + u32 *p; + + /* + * don't attempt to dump non-kernel addresses or + * values that are probably just small negative numbers + */ + if (addr < KIMAGE_VADDR || addr > -256UL) + return; + + printk("\n%s: %#lx:\n", name, addr); + + /* + * round address down to a 32 bit boundary + * and always dump a multiple of 32 bytes + */ + p = (u32 *)(addr & ~(sizeof(u32) - 1)); + nbytes += (addr & (sizeof(u32) - 1)); + nlines = (nbytes + 31) / 32; + + + for (i = 0; i < nlines; i++) { + /* + * just display low 16 bits of address to keep + * each line of the dump < 80 characters + */ + printk("%04lx ", (unsigned long)p & 0xffff); + for (j = 0; j < 8; j++) { + u32 data; + if (probe_kernel_address(p, data)) { + printk(" ********"); + } else { + pr_cont(" %08x", data); + } + ++p; + } + pr_cont("\n"); + } +} + +static void show_extra_register_data(struct pt_regs *regs, int nbytes) +{ + mm_segment_t fs; + + fs = get_fs(); + set_fs(KERNEL_DS); + show_data(regs->pc - nbytes, nbytes * 2, "PC"); + show_data(regs->regs[30] - nbytes, nbytes * 2, "LR"); + show_data(regs->sp - nbytes, nbytes * 2, "SP"); + set_fs(fs); +} + void __show_regs(struct pt_regs *regs) { int i, top_reg; @@ -190,6 +263,8 @@ if (i % 2 == 0) printk("\n"); } + if (!user_mode(regs)) + show_extra_register_data(regs, 64); printk("\n"); } @@ -289,6 +364,9 @@ } else { memset(childregs, 0, sizeof(struct pt_regs)); childregs->pstate = PSR_MODE_EL1h; + if (IS_ENABLED(CONFIG_ARM64_UAO) && + cpus_have_cap(ARM64_HAS_UAO)) + childregs->pstate |= PSR_UAO_BIT; p->thread.cpu_context.x19 = stack_start; p->thread.cpu_context.x20 = stk_sz; } @@ -302,20 +380,45 @@ static void tls_thread_switch(struct task_struct *next) { - unsigned long tpidr, tpidrro; + unsigned long tpidr; asm("mrs %0, tpidr_el0" : "=r" (tpidr)); *task_user_tls(current) = tpidr; - tpidr = *task_user_tls(next); - tpidrro = is_compat_thread(task_thread_info(next)) ? - next->thread.tp_value : 0; - - asm( - " msr tpidr_el0, %0\n" - " msr tpidrro_el0, %1" - : : "r" (tpidr), "r" (tpidrro)); + if (is_compat_thread(task_thread_info(next))) + write_sysreg(next->thread.tp_value, tpidrro_el0); + else if (!arm64_kernel_unmapped_at_el0()) + write_sysreg(0, tpidrro_el0); + + write_sysreg(*task_user_tls(next), tpidr_el0); +} + +/* Restore the UAO state depending on next's addr_limit */ +void uao_thread_switch(struct task_struct *next) +{ + if (IS_ENABLED(CONFIG_ARM64_UAO)) { + if (task_thread_info(next)->addr_limit == KERNEL_DS) + asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO)); + else + asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO)); + } +} + +#ifdef CONFIG_THREAD_INFO_IN_TASK +/* + * We store our current task in sp_el0, which is clobbered by userspace. Keep a + * shadow copy so that we can restore this upon entry from userspace. + * + * This is *only* for exception entry from EL0, and is not valid until we + * __switch_to() a user task. + */ +DEFINE_PER_CPU(struct task_struct *, __entry_task); + +static void entry_task_switch(struct task_struct *next) +{ + __this_cpu_write(__entry_task, next); } +#endif /* * Thread switching. @@ -329,6 +432,10 @@ tls_thread_switch(next); hw_breakpoint_thread_switch(next); contextidr_thread_switch(next); +#ifdef CONFIG_THREAD_INFO_IN_TASK + entry_task_switch(next); +#endif + uao_thread_switch(next); /* * Complete any pending TLB or cache maintenance on this CPU in case @@ -345,24 +452,35 @@ unsigned long get_wchan(struct task_struct *p) { struct stackframe frame; - unsigned long stack_page; + unsigned long stack_page, ret = 0; int count = 0; if (!p || p == current || p->state == TASK_RUNNING) return 0; + stack_page = (unsigned long)try_get_task_stack(p); + if (!stack_page) + return 0; + frame.fp = thread_saved_fp(p); frame.sp = thread_saved_sp(p); frame.pc = thread_saved_pc(p); - stack_page = (unsigned long)task_stack_page(p); +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + frame.graph = p->curr_ret_stack; +#endif do { if (frame.sp < stack_page || frame.sp >= stack_page + THREAD_SIZE || - unwind_frame(&frame)) - return 0; - if (!in_sched_functions(frame.pc)) - return frame.pc; + unwind_frame(p, &frame)) + goto out; + if (!in_sched_functions(frame.pc)) { + ret = frame.pc; + goto out; + } } while (count ++ < 16); - return 0; + +out: + put_task_stack(p); + return ret; } unsigned long arch_align_stack(unsigned long sp) diff -ruw linux-4.4.115/arch/arm64/kernel/psci.c linux-4.4.115-fbx/arch/arm64/kernel/psci.c --- linux-4.4.115/arch/arm64/kernel/psci.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/psci.c 2019-01-22 16:16:21.551228730 +0100 @@ -19,8 +19,8 @@ #include #include #include +#include #include -#include #include @@ -28,76 +28,10 @@ #include #include #include -#include - -static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state); - -static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) -{ - int i, ret, count = 0; - u32 *psci_states; - struct device_node *state_node, *cpu_node; - - cpu_node = of_get_cpu_node(cpu, NULL); - if (!cpu_node) - return -ENODEV; - - /* - * If the PSCI cpu_suspend function hook has not been initialized - * idle states must not be enabled, so bail out - */ - if (!psci_ops.cpu_suspend) - return -EOPNOTSUPP; - - /* Count idle states */ - while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states", - count))) { - count++; - of_node_put(state_node); - } - - if (!count) - return -ENODEV; - - psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL); - if (!psci_states) - return -ENOMEM; - - for (i = 0; i < count; i++) { - u32 state; - - state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i); - - ret = of_property_read_u32(state_node, - "arm,psci-suspend-param", - &state); - if (ret) { - pr_warn(" * %s missing arm,psci-suspend-param property\n", - state_node->full_name); - of_node_put(state_node); - goto free_mem; - } - - of_node_put(state_node); - pr_debug("psci-power-state %#x index %d\n", state, i); - if (!psci_power_state_is_valid(state)) { - pr_warn("Invalid PSCI power state %#x\n", state); - ret = -EINVAL; - goto free_mem; - } - psci_states[i] = state; - } - /* Idle states parsed correctly, initialize per-cpu pointer */ - per_cpu(psci_power_state, cpu) = psci_states; - return 0; - -free_mem: - kfree(psci_states); - return ret; -} static int __init cpu_psci_cpu_init(unsigned int cpu) { + pr_info("Initializing psci_cpu_init\n"); return 0; } @@ -113,7 +47,8 @@ static int cpu_psci_cpu_boot(unsigned int cpu) { - int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa(secondary_entry)); + int err = psci_ops.cpu_on(cpu_logical_map(cpu), + __pa_symbol(secondary_entry)); if (err) pr_err("failed to boot CPU%d (%d)\n", cpu, err); @@ -164,12 +99,12 @@ for (i = 0; i < 10; i++) { err = psci_ops.affinity_info(cpu_logical_map(cpu), 0); if (err == PSCI_0_2_AFFINITY_LEVEL_OFF) { - pr_info("CPU%d killed.\n", cpu); + pr_debug("CPU%d killed.\n", cpu); return 0; } msleep(10); - pr_info("Retrying again to check for CPU kill\n"); + pr_debug("Retrying again to check for CPU kill\n"); } pr_warn("CPU%d may not have shut down cleanly (AFFINITY_INFO reports %d)\n", @@ -178,38 +113,11 @@ } #endif -static int psci_suspend_finisher(unsigned long index) -{ - u32 *state = __this_cpu_read(psci_power_state); - - return psci_ops.cpu_suspend(state[index - 1], - virt_to_phys(cpu_resume)); -} - -static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index) -{ - int ret; - u32 *state = __this_cpu_read(psci_power_state); - /* - * idle state index 0 corresponds to wfi, should never be called - * from the cpu_suspend operations - */ - if (WARN_ON_ONCE(!index)) - return -EINVAL; - - if (!psci_power_state_loses_context(state[index - 1])) - ret = psci_ops.cpu_suspend(state[index - 1], 0); - else - ret = cpu_suspend(index, psci_suspend_finisher); - - return ret; -} - const struct cpu_operations cpu_psci_ops = { .name = "psci", #ifdef CONFIG_CPU_IDLE - .cpu_init_idle = cpu_psci_cpu_init_idle, - .cpu_suspend = cpu_psci_cpu_suspend, + .cpu_init_idle = psci_cpu_init_idle, + .cpu_suspend = psci_cpu_suspend_enter, #endif .cpu_init = cpu_psci_cpu_init, .cpu_prepare = cpu_psci_cpu_prepare, diff -ruw linux-4.4.115/arch/arm64/kernel/ptrace.c linux-4.4.115-fbx/arch/arm64/kernel/ptrace.c --- linux-4.4.115/arch/arm64/kernel/ptrace.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/ptrace.c 2019-10-29 09:26:23.005196935 +0100 @@ -49,6 +49,106 @@ #define CREATE_TRACE_POINTS #include +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} +#define GPR_OFFSET_NAME(r) \ + {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])} + +static const struct pt_regs_offset regoffset_table[] = { + GPR_OFFSET_NAME(0), + GPR_OFFSET_NAME(1), + GPR_OFFSET_NAME(2), + GPR_OFFSET_NAME(3), + GPR_OFFSET_NAME(4), + GPR_OFFSET_NAME(5), + GPR_OFFSET_NAME(6), + GPR_OFFSET_NAME(7), + GPR_OFFSET_NAME(8), + GPR_OFFSET_NAME(9), + GPR_OFFSET_NAME(10), + GPR_OFFSET_NAME(11), + GPR_OFFSET_NAME(12), + GPR_OFFSET_NAME(13), + GPR_OFFSET_NAME(14), + GPR_OFFSET_NAME(15), + GPR_OFFSET_NAME(16), + GPR_OFFSET_NAME(17), + GPR_OFFSET_NAME(18), + GPR_OFFSET_NAME(19), + GPR_OFFSET_NAME(20), + GPR_OFFSET_NAME(21), + GPR_OFFSET_NAME(22), + GPR_OFFSET_NAME(23), + GPR_OFFSET_NAME(24), + GPR_OFFSET_NAME(25), + GPR_OFFSET_NAME(26), + GPR_OFFSET_NAME(27), + GPR_OFFSET_NAME(28), + GPR_OFFSET_NAME(29), + GPR_OFFSET_NAME(30), + {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])}, + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(pc), + REG_OFFSET_NAME(pstate), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + return ((addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + /* * TODO: does not yet catch signals sent when the child dies. * in exit.c or in signal.c. @@ -227,13 +327,13 @@ struct arch_hw_breakpoint_ctrl ctrl, struct perf_event_attr *attr) { - int err, len, type, disabled = !ctrl.enabled; + int err, len, type, offset, disabled = !ctrl.enabled; attr->disabled = disabled; if (disabled) return 0; - err = arch_bp_generic_fields(ctrl, &len, &type); + err = arch_bp_generic_fields(ctrl, &len, &type, &offset); if (err) return err; @@ -252,6 +352,7 @@ attr->bp_len = len; attr->bp_type = type; + attr->bp_addr += offset; return 0; } @@ -304,7 +405,7 @@ if (IS_ERR(bp)) return PTR_ERR(bp); - *addr = bp ? bp->attr.bp_addr : 0; + *addr = bp ? counter_arch_bp(bp)->address : 0; return 0; } diff -ruw linux-4.4.115/arch/arm64/kernel/return_address.c linux-4.4.115-fbx/arch/arm64/kernel/return_address.c --- linux-4.4.115/arch/arm64/kernel/return_address.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/return_address.c 2019-01-22 16:16:21.555228766 +0100 @@ -12,6 +12,7 @@ #include #include +#include #include struct return_address_data { @@ -43,8 +44,11 @@ frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.pc = (unsigned long)return_address; /* dummy */ +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + frame.graph = current->curr_ret_stack; +#endif - walk_stackframe(&frame, save_return_addr, &data); + walk_stackframe(current, &frame, save_return_addr, &data); if (!data.level) return data.addr; diff -ruw linux-4.4.115/arch/arm64/kernel/setup.c linux-4.4.115-fbx/arch/arm64/kernel/setup.c --- linux-4.4.115/arch/arm64/kernel/setup.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/setup.c 2019-01-22 16:16:21.555228766 +0100 @@ -44,6 +44,8 @@ #include #include #include +#include +#include #include #include @@ -62,9 +64,17 @@ #include #include #include +#include + +unsigned int boot_reason; +EXPORT_SYMBOL(boot_reason); + +unsigned int cold_boot; +EXPORT_SYMBOL(cold_boot); phys_addr_t __fdt_pointer __initdata; +const char *machine_name; /* * Standard memory resources */ @@ -174,7 +184,6 @@ */ if (mpidr_hash_size() > 4 * num_possible_cpus()) pr_warn("Large number of MPIDR hash buckets detected\n"); - __flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash)); } static void __init setup_machine_fdt(phys_addr_t dt_phys) @@ -192,7 +201,11 @@ cpu_relax(); } - dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name()); + machine_name = of_flat_dt_get_machine_name(); + if (machine_name) { + dump_stack_set_arch_desc("%s (DT)", machine_name); + pr_info("Machine: %s\n", machine_name); + } } static void __init request_standard_resources(void) @@ -200,10 +213,10 @@ struct memblock_region *region; struct resource *res; - kernel_code.start = virt_to_phys(_text); - kernel_code.end = virt_to_phys(_etext - 1); - kernel_data.start = virt_to_phys(_sdata); - kernel_data.end = virt_to_phys(_end - 1); + kernel_code.start = __pa_symbol(_text); + kernel_code.end = __pa_symbol(__init_begin - 1); + kernel_data.start = __pa_symbol(_sdata); + kernel_data.end = __pa_symbol(_end - 1); for_each_memblock(memory, region) { res = alloc_bootmem_low(sizeof(*res)); @@ -288,6 +301,8 @@ u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID }; +void __init __weak init_random_pool(void) { } + void __init setup_arch(char **cmdline_p) { pr_info("Boot CPU: AArch64 Processor [%08x]\n", read_cpuid_id()); @@ -313,6 +328,12 @@ */ local_async_enable(); + /* + * TTBR0 is only used for the identity mapping at this stage. Make it + * point to zero page to avoid speculatively fetching new entries. + */ + cpu_uninstall_idmap(); + efi_init(); arm64_memblock_init(); @@ -340,6 +361,19 @@ smp_init_cpus(); smp_build_mpidr_hash(); +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + /* + * Make sure thread_info.ttbr0 always generates translation + * faults in case uaccess_enable() is inadvertently called by the init + * thread. + */ +#ifdef CONFIG_THREAD_INFO_IN_TASK + init_task.thread_info.ttbr0 = __pa_symbol(empty_zero_page); +#else + init_thread_info.ttbr0 = __pa_symbol(empty_zero_page); +#endif +#endif + #ifdef CONFIG_VT #if defined(CONFIG_VGA_CONSOLE) conswitchp = &vga_con; @@ -347,6 +381,7 @@ conswitchp = &dummy_con; #endif #endif + init_random_pool(); if (boot_args[1] || boot_args[2] || boot_args[3]) { pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" "\tx1: %016llx\n\tx2: %016llx\n\tx3: %016llx\n" @@ -380,4 +415,39 @@ return 0; } -subsys_initcall(topology_init); +postcore_initcall(topology_init); + +void arch_setup_pdev_archdata(struct platform_device *pdev) +{ + pdev->archdata.dma_mask = DMA_BIT_MASK(32); + pdev->dev.dma_mask = &pdev->archdata.dma_mask; +} + +/* + * Dump out kernel offset information on panic. + */ +static int dump_kernel_offset(struct notifier_block *self, unsigned long v, + void *p) +{ + const unsigned long offset = kaslr_offset(); + + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && offset > 0) { + pr_emerg("Kernel Offset: 0x%lx from 0x%lx\n", + offset, KIMAGE_VADDR); + } else { + pr_emerg("Kernel Offset: disabled\n"); + } + return 0; +} + +static struct notifier_block kernel_offset_notifier = { + .notifier_call = dump_kernel_offset +}; + +static int __init register_kernel_offset_dumper(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &kernel_offset_notifier); + return 0; +} +__initcall(register_kernel_offset_dumper); diff -ruw linux-4.4.115/arch/arm64/kernel/sleep.S linux-4.4.115-fbx/arch/arm64/kernel/sleep.S --- linux-4.4.115/arch/arm64/kernel/sleep.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/sleep.S 2019-10-29 09:26:23.005196935 +0100 @@ -49,39 +49,32 @@ orr \dst, \dst, \mask // dst|=(aff3>>rs3) .endm /* - * Save CPU state for a suspend and execute the suspend finisher. - * On success it will return 0 through cpu_resume - ie through a CPU - * soft/hard reboot from the reset vector. - * On failure it returns the suspend finisher return value or force - * -EOPNOTSUPP if the finisher erroneously returns 0 (the suspend finisher - * is not allowed to return, if it does this must be considered failure). - * It saves callee registers, and allocates space on the kernel stack - * to save the CPU specific registers + some other data for resume. + * Save CPU state in the provided sleep_stack_data area, and publish its + * location for cpu_resume()'s use in sleep_save_stash. * - * x0 = suspend finisher argument - * x1 = suspend finisher function pointer + * cpu_resume() will restore this saved state, and return. Because the + * link-register is saved and restored, it will appear to return from this + * function. So that the caller can tell the suspend/resume paths apart, + * __cpu_suspend_enter() will always return a non-zero value, whereas the + * path through cpu_resume() will return 0. + * + * x0 = struct sleep_stack_data area */ ENTRY(__cpu_suspend_enter) - stp x29, lr, [sp, #-96]! - stp x19, x20, [sp,#16] - stp x21, x22, [sp,#32] - stp x23, x24, [sp,#48] - stp x25, x26, [sp,#64] - stp x27, x28, [sp,#80] - /* - * Stash suspend finisher and its argument in x20 and x19 - */ - mov x19, x0 - mov x20, x1 + stp x29, lr, [x0, #SLEEP_STACK_DATA_CALLEE_REGS] + stp x19, x20, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+16] + stp x21, x22, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+32] + stp x23, x24, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+48] + stp x25, x26, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+64] + stp x27, x28, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+80] + + /* save the sp in cpu_suspend_ctx */ mov x2, sp - sub sp, sp, #CPU_SUSPEND_SZ // allocate cpu_suspend_ctx - mov x0, sp - /* - * x0 now points to struct cpu_suspend_ctx allocated on the stack - */ - str x2, [x0, #CPU_CTX_SP] - ldr x1, =sleep_save_sp - ldr x1, [x1, #SLEEP_SAVE_SP_VIRT] + str x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP] + + /* find the mpidr_hash */ + ldr x1, =sleep_save_stash + ldr x1, [x1] mrs x7, mpidr_el1 ldr x9, =mpidr_hash ldr x10, [x9, #MPIDR_HASH_MASK] @@ -93,70 +86,28 @@ ldp w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)] compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10 add x1, x1, x8, lsl #3 - bl __cpu_suspend_save - /* - * Grab suspend finisher in x20 and its argument in x19 - */ - mov x0, x19 - mov x1, x20 - /* - * We are ready for power down, fire off the suspend finisher - * in x1, with argument in x0 - */ - blr x1 - /* - * Never gets here, unless suspend finisher fails. - * Successful cpu_suspend should return from cpu_resume, returning - * through this code path is considered an error - * If the return value is set to 0 force x0 = -EOPNOTSUPP - * to make sure a proper error condition is propagated - */ - cmp x0, #0 - mov x3, #-EOPNOTSUPP - csel x0, x3, x0, eq - add sp, sp, #CPU_SUSPEND_SZ // rewind stack pointer - ldp x19, x20, [sp, #16] - ldp x21, x22, [sp, #32] - ldp x23, x24, [sp, #48] - ldp x25, x26, [sp, #64] - ldp x27, x28, [sp, #80] - ldp x29, lr, [sp], #96 + + str x0, [x1] + add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS + stp x29, lr, [sp, #-16]! + bl cpu_do_suspend + ldp x29, lr, [sp], #16 + mov x0, #1 ret ENDPROC(__cpu_suspend_enter) .ltorg -/* - * x0 must contain the sctlr value retrieved from restored context - */ - .pushsection ".idmap.text", "ax" -ENTRY(cpu_resume_mmu) - ldr x3, =cpu_resume_after_mmu - msr sctlr_el1, x0 // restore sctlr_el1 - isb - /* - * Invalidate the local I-cache so that any instructions fetched - * speculatively from the PoC are discarded, since they may have - * been dynamically patched at the PoU. - */ - ic iallu - dsb nsh - isb - br x3 // global jump to virtual address -ENDPROC(cpu_resume_mmu) - .popsection -cpu_resume_after_mmu: - mov x0, #0 // return zero on success - ldp x19, x20, [sp, #16] - ldp x21, x22, [sp, #32] - ldp x23, x24, [sp, #48] - ldp x25, x26, [sp, #64] - ldp x27, x28, [sp, #80] - ldp x29, lr, [sp], #96 - ret -ENDPROC(cpu_resume_after_mmu) - ENTRY(cpu_resume) bl el2_setup // if in EL2 drop to EL1 cleanly + /* enable the MMU early - so we can access sleep_save_stash by va */ + adr_l lr, __enable_mmu /* __cpu_setup will return here */ + ldr x27, =_cpu_resume /* __enable_mmu will branch here */ + adrp x25, idmap_pg_dir + adrp x26, swapper_pg_dir + b __cpu_setup +ENDPROC(cpu_resume) + +ENTRY(_cpu_resume) mrs x1, mpidr_el1 adrp x8, mpidr_hash add x8, x8, #:lo12:mpidr_hash // x8 = struct mpidr_hash phys address @@ -166,17 +117,29 @@ ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)] compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2 /* x7 contains hash index, let's use it to grab context pointer */ - ldr_l x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS + ldr_l x0, sleep_save_stash ldr x0, [x0, x7, lsl #3] + add x29, x0, #SLEEP_STACK_DATA_CALLEE_REGS + add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS /* load sp from context */ ldr x2, [x0, #CPU_CTX_SP] - /* load physical address of identity map page table in x1 */ - adrp x1, idmap_pg_dir mov sp, x2 /* - * cpu_do_resume expects x0 to contain context physical address - * pointer and x1 to contain physical address of 1:1 page tables + * cpu_do_resume expects x0 to contain context address pointer */ - bl cpu_do_resume // PC relative jump, MMU off - b cpu_resume_mmu // Resume MMU, never returns -ENDPROC(cpu_resume) + bl cpu_do_resume + +#ifdef CONFIG_KASAN + mov x0, sp + bl kasan_unpoison_task_stack_below +#endif + + ldp x19, x20, [x29, #16] + ldp x21, x22, [x29, #32] + ldp x23, x24, [x29, #48] + ldp x25, x26, [x29, #64] + ldp x27, x28, [x29, #80] + ldp x29, lr, [x29] + mov x0, #0 + ret +ENDPROC(_cpu_resume) diff -ruw linux-4.4.115/arch/arm64/kernel/smp.c linux-4.4.115-fbx/arch/arm64/kernel/smp.c --- linux-4.4.115/arch/arm64/kernel/smp.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/smp.c 2019-10-29 09:26:23.005196935 +0100 @@ -53,10 +53,15 @@ #include #include #include +#include +#include #define CREATE_TRACE_POINTS #include +DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number); +EXPORT_PER_CPU_SYMBOL(cpu_number); + /* * as from 2.5, kernels no longer have an init_tasks structure * so we need some other way of telling a new secondary core @@ -70,6 +75,8 @@ IPI_CPU_STOP, IPI_TIMER, IPI_IRQ_WORK, + IPI_WAKEUP, + IPI_CPU_BACKTRACE, }; /* @@ -94,6 +101,9 @@ * We need to tell the secondary core where to find its stack and the * page tables. */ +#ifdef CONFIG_THREAD_INFO_IN_TASK + secondary_data.task = idle; +#endif secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; __flush_dcache_area(&secondary_data, sizeof(secondary_data)); @@ -117,6 +127,9 @@ pr_err("CPU%u: failed to boot: %d\n", cpu, ret); } +#ifdef CONFIG_THREAD_INFO_IN_TASK + secondary_data.task = NULL; +#endif secondary_data.stack = NULL; return ret; @@ -134,7 +147,12 @@ asmlinkage void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; - unsigned int cpu = smp_processor_id(); + unsigned int cpu; + + cpu = task_cpu(current); + set_my_cpu_offset(per_cpu_offset(cpu)); + + pr_debug("CPU%u: Booted secondary processor\n", cpu); /* * All kernel threads share the same mm context; grab a @@ -143,15 +161,11 @@ atomic_inc(&mm->mm_count); current->active_mm = mm; - set_my_cpu_offset(per_cpu_offset(smp_processor_id())); - /* * TTBR0 is only used for the identity mapping at this stage. Make it * point to zero page to avoid speculatively fetching new entries. */ - cpu_set_reserved_ttbr0(); - local_flush_tlb_all(); - cpu_set_default_tcr_t0sz(); + cpu_uninstall_idmap(); preempt_disable(); trace_hardirqs_off(); @@ -174,10 +188,10 @@ /* * Enable GIC and timers. */ - notify_cpu_starting(cpu); - smp_store_cpu_info(cpu); + notify_cpu_starting(cpu); + /* * OK, now it's safe to let the boot CPU continue. Wait for * the CPU migration code to notice that the CPU is online @@ -268,7 +282,7 @@ pr_crit("CPU%u: cpu didn't die\n", cpu); return; } - pr_notice("CPU%u: shutdown\n", cpu); + pr_debug("CPU%u: shutdown\n", cpu); /* * Now that the dying CPU is beyond the point of no return w.r.t. @@ -290,7 +304,7 @@ * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */ -void cpu_die(void) +void __ref cpu_die(void) { unsigned int cpu = smp_processor_id(); @@ -308,7 +322,16 @@ */ cpu_ops[cpu]->cpu_die(cpu); - BUG(); + /* + * Do not return to the idle loop - jump back to the secondary + * cpu initialisation. There's some initialisation which needs + * to be repeated to undo the effects of taking the CPU offline. + */ + + asm volatile("mov sp, %0\n" + "mov x29, #0\n" + "b secondary_start_kernel" + : : "r" (task_stack_page(current) + THREAD_START_SP)); } #endif @@ -444,6 +467,17 @@ /* map the logical cpu id to cpu MPIDR */ cpu_logical_map(cpu_count) = hwid; + /* + * Set-up the ACPI parking protocol cpu entries + * while initializing the cpu_logical_map to + * avoid parsing MADT entries multiple times for + * nothing (ie a valid cpu_logical_map entry should + * contain a valid parking protocol data set to + * initialize the cpu if the parking protocol is + * the only available enable method). + */ + acpi_set_mailbox_entry(cpu_count, processor); + cpu_count++; } @@ -466,6 +500,18 @@ #else #define acpi_table_parse_madt(...) do { } while (0) #endif +void (*__smp_cross_call)(const struct cpumask *, unsigned int); +DEFINE_PER_CPU(bool, pending_ipi); + +void smp_cross_call_common(const struct cpumask *cpumask, unsigned int func) +{ + unsigned int cpu; + + for_each_cpu(cpu, cpumask) + per_cpu(pending_ipi, cpu) = true; + + __smp_cross_call(cpumask, func); +} /* * Enumerate the possible CPU set from the device tree and build the @@ -597,6 +643,8 @@ if (max_cpus == 0) break; + per_cpu(cpu_number, cpu) = cpu; + if (cpu == smp_processor_id()) continue; @@ -612,8 +660,6 @@ } } -void (*__smp_cross_call)(const struct cpumask *, unsigned int); - void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) { __smp_cross_call = fn; @@ -626,10 +672,17 @@ S(IPI_CPU_STOP, "CPU stop interrupts"), S(IPI_TIMER, "Timer broadcast interrupts"), S(IPI_IRQ_WORK, "IRQ work interrupts"), + S(IPI_WAKEUP, "CPU wakeup interrupts"), + S(IPI_CPU_BACKTRACE, "CPU backtrace"), }; static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) { + unsigned int cpu; + + for_each_cpu(cpu, target) + per_cpu(pending_ipi, cpu) = true; + trace_ipi_raise(target, ipi_types[ipinr]); __smp_cross_call(target, ipinr); } @@ -661,14 +714,21 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask) { - smp_cross_call(mask, IPI_CALL_FUNC); + smp_cross_call_common(mask, IPI_CALL_FUNC); } void arch_send_call_function_single_ipi(int cpu) { - smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC); + smp_cross_call_common(cpumask_of(cpu), IPI_CALL_FUNC); } +#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL +void arch_send_wakeup_ipi_mask(const struct cpumask *mask) +{ + smp_cross_call(mask, IPI_WAKEUP); +} +#endif + #ifdef CONFIG_IRQ_WORK void arch_irq_work_raise(void) { @@ -677,29 +737,86 @@ } #endif -static DEFINE_RAW_SPINLOCK(stop_lock); - /* * ipi_cpu_stop - handle IPI from smp_send_stop() */ static void ipi_cpu_stop(unsigned int cpu) { - if (system_state == SYSTEM_BOOTING || - system_state == SYSTEM_RUNNING) { - raw_spin_lock(&stop_lock); - pr_crit("CPU%u: stopping\n", cpu); - dump_stack(); - raw_spin_unlock(&stop_lock); - } - - set_cpu_online(cpu, false); + set_cpu_active(cpu, false); + flush_cache_all(); local_irq_disable(); while (1) cpu_relax(); } +static cpumask_t backtrace_mask; +static DEFINE_RAW_SPINLOCK(backtrace_lock); + +/* "in progress" flag of arch_trigger_all_cpu_backtrace */ +static unsigned long backtrace_flag; + +static void smp_send_all_cpu_backtrace(void) +{ + unsigned int this_cpu = smp_processor_id(); + int i; + + if (test_and_set_bit(0, &backtrace_flag)) + /* + * If there is already a trigger_all_cpu_backtrace() in progress + * (backtrace_flag == 1), don't output double cpu dump infos. + */ + return; + + cpumask_copy(&backtrace_mask, cpu_online_mask); + cpumask_clear_cpu(this_cpu, &backtrace_mask); + + pr_info("Backtrace for cpu %d (current):\n", this_cpu); + dump_stack(); + + pr_info("\nsending IPI to all other CPUs:\n"); + if (!cpumask_empty(&backtrace_mask)) + smp_cross_call_common(&backtrace_mask, IPI_CPU_BACKTRACE); + + /* Wait for up to 10 seconds for all other CPUs to do the backtrace */ + for (i = 0; i < 10 * 1000; i++) { + if (cpumask_empty(&backtrace_mask)) + break; + mdelay(1); + } + + clear_bit(0, &backtrace_flag); + smp_mb__after_atomic(); +} + +/* + * ipi_cpu_backtrace - handle IPI from smp_send_all_cpu_backtrace() + */ +static void ipi_cpu_backtrace(unsigned int cpu, struct pt_regs *regs) +{ + if (cpumask_test_cpu(cpu, &backtrace_mask)) { + raw_spin_lock(&backtrace_lock); + pr_warn("IPI backtrace for cpu %d\n", cpu); + show_regs(regs); + raw_spin_unlock(&backtrace_lock); + cpumask_clear_cpu(cpu, &backtrace_mask); + } +} + +#ifdef CONFIG_SMP +void arch_trigger_all_cpu_backtrace(void) +{ + smp_send_all_cpu_backtrace(); +} +#else +void arch_trigger_all_cpu_backtrace(void) +{ + dump_stack(); +} +#endif + + /* * Main handler for inter-processor interrupts */ @@ -746,6 +863,18 @@ break; #endif + case IPI_CPU_BACKTRACE: + ipi_cpu_backtrace(cpu, regs); + break; + +#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL + case IPI_WAKEUP: + WARN_ONCE(!acpi_parking_protocol_valid(cpu), + "CPU%u: Wake-up IPI outside the ACPI parking protocol\n", + cpu); + break; +#endif + default: pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); break; @@ -753,18 +882,21 @@ if ((unsigned)ipinr < NR_IPI) trace_ipi_exit_rcuidle(ipi_types[ipinr]); + + per_cpu(pending_ipi, cpu) = false; set_irq_regs(old_regs); } void smp_send_reschedule(int cpu) { - smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); + BUG_ON(cpu_is_offline(cpu)); + smp_cross_call_common(cpumask_of(cpu), IPI_RESCHEDULE); } #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST void tick_broadcast(const struct cpumask *mask) { - smp_cross_call(mask, IPI_TIMER); + smp_cross_call_common(mask, IPI_TIMER); } #endif @@ -778,16 +910,20 @@ cpumask_copy(&mask, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &mask); - smp_cross_call(&mask, IPI_CPU_STOP); + if (system_state == SYSTEM_BOOTING || + system_state == SYSTEM_RUNNING) + pr_crit("SMP: stopping secondary CPUs\n"); + smp_cross_call_common(&mask, IPI_CPU_STOP); } /* Wait up to one second for other CPUs to stop */ timeout = USEC_PER_SEC; - while (num_online_cpus() > 1 && timeout--) + while (num_active_cpus() > 1 && timeout--) udelay(1); - if (num_online_cpus() > 1) - pr_warning("SMP: failed to stop secondary CPUs\n"); + if (num_active_cpus() > 1) + pr_warning("SMP: failed to stop secondary CPUs %*pbl\n", + cpumask_pr_args(cpu_online_mask)); } /* diff -ruw linux-4.4.115/arch/arm64/kernel/smp_spin_table.c linux-4.4.115-fbx/arch/arm64/kernel/smp_spin_table.c --- linux-4.4.115/arch/arm64/kernel/smp_spin_table.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/smp_spin_table.c 2019-10-29 09:26:23.005196935 +0100 @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -96,7 +97,7 @@ * boot-loader's endianess before jumping. This is mandated by * the boot protocol. */ - writeq_relaxed(__pa(secondary_holding_pen), release_addr); + writeq_relaxed(__pa_symbol(secondary_holding_pen), release_addr); __flush_dcache_area((__force void *)release_addr, sizeof(*release_addr)); diff -ruw linux-4.4.115/arch/arm64/kernel/stacktrace.c linux-4.4.115-fbx/arch/arm64/kernel/stacktrace.c --- linux-4.4.115/arch/arm64/kernel/stacktrace.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/stacktrace.c 2019-01-22 16:16:21.555228766 +0100 @@ -15,11 +15,15 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ +#include #include #include +#include #include #include +#include +#include #include /* @@ -35,25 +39,86 @@ * ldp x29, x30, [sp] * add sp, sp, #0x10 */ -int notrace unwind_frame(struct stackframe *frame) +int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) { unsigned long high, low; unsigned long fp = frame->fp; + unsigned long irq_stack_ptr; + + /* + * Switching between stacks is valid when tracing current and in + * non-preemptible context. + */ + if (tsk == current && !preemptible()) + irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id()); + else + irq_stack_ptr = 0; low = frame->sp; - high = ALIGN(low, THREAD_SIZE); + /* irq stacks are not THREAD_SIZE aligned */ + if (on_irq_stack(frame->sp, raw_smp_processor_id())) + high = irq_stack_ptr; + else + high = ALIGN(low, THREAD_SIZE) - 0x20; - if (fp < low || fp > high - 0x18 || fp & 0xf) + if (fp < low || fp > high || fp & 0xf) return -EINVAL; + kasan_disable_current(); + frame->sp = fp + 0x10; frame->fp = *(unsigned long *)(fp); frame->pc = *(unsigned long *)(fp + 8); + kasan_enable_current(); + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + if (tsk && tsk->ret_stack && + (frame->pc == (unsigned long)return_to_handler)) { + /* + * This is a case where function graph tracer has + * modified a return address (LR) in a stack frame + * to hook a function return. + * So replace it to an original value. + */ + frame->pc = tsk->ret_stack[frame->graph--].ret; + } +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + + /* + * Check whether we are going to walk through from interrupt stack + * to task stack. + * If we reach the end of the stack - and its an interrupt stack, + * unpack the dummy frame to find the original elr. + * + * Check the frame->fp we read from the bottom of the irq_stack, + * and the original task stack pointer are both in current->stack. + */ + if (frame->sp == irq_stack_ptr) { + struct pt_regs *irq_args; + unsigned long orig_sp = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); + + if (object_is_on_stack((void *)orig_sp) && + object_is_on_stack((void *)frame->fp)) { + frame->sp = orig_sp; + + /* orig_sp is the saved pt_regs, find the elr */ + irq_args = (struct pt_regs *)orig_sp; + frame->pc = irq_args->pc; + } else { + /* + * This frame has a non-standard format, and we + * didn't fix it, because the data looked wrong. + * Refuse to output this frame. + */ + return -EINVAL; + } + } + return 0; } -void notrace walk_stackframe(struct stackframe *frame, +void notrace walk_stackframe(struct task_struct *tsk, struct stackframe *frame, int (*fn)(struct stackframe *, void *), void *data) { while (1) { @@ -61,12 +126,11 @@ if (fn(frame, data)) break; - ret = unwind_frame(frame); + ret = unwind_frame(tsk, frame); if (ret < 0) break; } } -EXPORT_SYMBOL(walk_stackframe); #ifdef CONFIG_STACKTRACE struct stack_trace_data { @@ -93,34 +157,51 @@ return trace->nr_entries >= trace->max_entries; } -void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +static noinline void __save_stack_trace(struct task_struct *tsk, + struct stack_trace *trace, unsigned int nosched) { struct stack_trace_data data; struct stackframe frame; + if (!try_get_task_stack(tsk)) + return; + data.trace = trace; data.skip = trace->skip; + data.no_sched_functions = nosched; if (tsk != current) { - data.no_sched_functions = 1; frame.fp = thread_saved_fp(tsk); frame.sp = thread_saved_sp(tsk); frame.pc = thread_saved_pc(tsk); } else { - data.no_sched_functions = 0; + /* We don't want this function nor the caller */ + data.skip += 2; frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; - frame.pc = (unsigned long)save_stack_trace_tsk; + frame.pc = (unsigned long)__save_stack_trace; } +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + frame.graph = tsk->curr_ret_stack; +#endif - walk_stackframe(&frame, save_trace, &data); + walk_stackframe(tsk, &frame, save_trace, &data); if (trace->nr_entries < trace->max_entries) trace->entries[trace->nr_entries++] = ULONG_MAX; + + put_task_stack(tsk); +} +EXPORT_SYMBOL(save_stack_trace_tsk); + +void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +{ + __save_stack_trace(tsk, trace, 1); } void save_stack_trace(struct stack_trace *trace) { - save_stack_trace_tsk(current, trace); + __save_stack_trace(current, trace, 0); } + EXPORT_SYMBOL_GPL(save_stack_trace); #endif diff -ruw linux-4.4.115/arch/arm64/kernel/suspend.c linux-4.4.115-fbx/arch/arm64/kernel/suspend.c --- linux-4.4.115/arch/arm64/kernel/suspend.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/suspend.c 2019-01-22 16:16:21.555228766 +0100 @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -12,30 +13,11 @@ #include #include -extern int __cpu_suspend_enter(unsigned long arg, int (*fn)(unsigned long)); /* - * This is called by __cpu_suspend_enter() to save the state, and do whatever - * flushing is required to ensure that when the CPU goes to sleep we have - * the necessary data available when the caches are not searched. - * - * ptr: CPU context virtual address - * save_ptr: address of the location where the context physical address - * must be saved - */ -void notrace __cpu_suspend_save(struct cpu_suspend_ctx *ptr, - phys_addr_t *save_ptr) -{ - *save_ptr = virt_to_phys(ptr); - - cpu_do_suspend(ptr); - /* - * Only flush the context that must be retrieved with the MMU - * off. VA primitives ensure the flush is applied to all - * cache levels so context is pushed to DRAM. + * This is allocated by cpu_suspend_init(), and used to store a pointer to + * the 'struct sleep_stack_data' the contains a particular CPUs state. */ - __flush_dcache_area(ptr, sizeof(*ptr)); - __flush_dcache_area(save_ptr, sizeof(*save_ptr)); -} +unsigned long *sleep_save_stash; /* * This hook is provided so that cpu_suspend code can restore HW @@ -53,6 +35,24 @@ hw_breakpoint_restore = hw_bp_restore; } +void notrace __cpu_suspend_exit(void) +{ + /* + * We are resuming from reset with the idmap active in TTBR0_EL1. + * We must uninstall the idmap and restore the expected MMU + * state before we can possibly return to userspace. + */ + cpu_uninstall_idmap(); + + /* + * Restore HW breakpoint registers to sane values + * before debug exceptions are possibly reenabled + * through local_dbg_restore. + */ + if (hw_breakpoint_restore) + hw_breakpoint_restore(NULL); +} + /* * cpu_suspend * @@ -62,9 +62,9 @@ */ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) { - struct mm_struct *mm = current->active_mm; - int ret; + int ret = 0; unsigned long flags; + struct sleep_stack_data state; /* * From this point debug exceptions are disabled to prevent @@ -80,37 +80,9 @@ */ pause_graph_tracing(); - /* - * mm context saved on the stack, it will be restored when - * the cpu comes out of reset through the identity mapped - * page tables, so that the thread address space is properly - * set-up on function return. - */ - ret = __cpu_suspend_enter(arg, fn); - if (ret == 0) { - /* - * We are resuming from reset with TTBR0_EL1 set to the - * idmap to enable the MMU; set the TTBR0 to the reserved - * page tables to prevent speculative TLB allocations, flush - * the local tlb and set the default tcr_el1.t0sz so that - * the TTBR0 address space set-up is properly restored. - * If the current active_mm != &init_mm we entered cpu_suspend - * with mappings in TTBR0 that must be restored, so we switch - * them back to complete the address space configuration - * restoration before returning. - */ - cpu_set_reserved_ttbr0(); - local_flush_tlb_all(); - cpu_set_default_tcr_t0sz(); - - if (mm != &init_mm) - cpu_switch_mm(mm->pgd, mm); - - /* - * Restore per-cpu offset before any kernel - * subsystem relying on it has a chance to run. - */ - set_my_cpu_offset(per_cpu_offset(smp_processor_id())); + if (__cpu_suspend_enter(&state)) { + /* Call the suspend finisher */ + ret = fn(arg); /* * PSTATE was not saved over suspend/resume, re-enable any @@ -118,14 +90,17 @@ */ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)); + uao_thread_switch(current); /* * Restore HW breakpoint registers to sane values * before debug exceptions are possibly reenabled * through local_dbg_restore. */ - if (hw_breakpoint_restore) - hw_breakpoint_restore(NULL); + if (!ret) + ret = -EOPNOTSUPP; + } else { + __cpu_suspend_exit(); } unpause_graph_tracing(); @@ -140,22 +115,15 @@ return ret; } -struct sleep_save_sp sleep_save_sp; - static int __init cpu_suspend_init(void) { - void *ctx_ptr; - /* ctx_ptr is an array of physical addresses */ - ctx_ptr = kcalloc(mpidr_hash_size(), sizeof(phys_addr_t), GFP_KERNEL); + sleep_save_stash = kcalloc(mpidr_hash_size(), sizeof(*sleep_save_stash), + GFP_KERNEL); - if (WARN_ON(!ctx_ptr)) + if (WARN_ON(!sleep_save_stash)) return -ENOMEM; - sleep_save_sp.save_ptr_stash = ctx_ptr; - sleep_save_sp.save_ptr_stash_phys = virt_to_phys(ctx_ptr); - __flush_dcache_area(&sleep_save_sp, sizeof(struct sleep_save_sp)); - return 0; } early_initcall(cpu_suspend_init); diff -ruw linux-4.4.115/arch/arm64/kernel/time.c linux-4.4.115-fbx/arch/arm64/kernel/time.c --- linux-4.4.115/arch/arm64/kernel/time.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/time.c 2019-01-22 16:16:21.555228766 +0100 @@ -52,8 +52,11 @@ frame.fp = regs->regs[29]; frame.sp = regs->sp; frame.pc = regs->pc; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + frame.graph = -1; /* no task info */ +#endif do { - int ret = unwind_frame(&frame); + int ret = unwind_frame(NULL, &frame); if (ret < 0) return 0; } while (in_lock_functions(frame.pc)); diff -ruw linux-4.4.115/arch/arm64/kernel/topology.c linux-4.4.115-fbx/arch/arm64/kernel/topology.c --- linux-4.4.115/arch/arm64/kernel/topology.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/topology.c 2019-01-22 16:16:21.555228766 +0100 @@ -19,10 +19,52 @@ #include #include #include +#include +#include +#include #include #include +/* + * cpu power table + * This per cpu data structure describes the relative capacity of each core. + * On a heteregenous system, cores don't have the same computation capacity + * and we reflect that difference in the cpu_power field so the scheduler can + * take this difference into account during load balance. A per cpu structure + * is preferred because each CPU updates its own cpu_power field during the + * load balance except for idle cores. One idle core is selected to run the + * rebalance_domains for all idle cores and the cpu_power can be updated + * during this sequence. + */ +static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; + +unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu) +{ + return per_cpu(cpu_scale, cpu); +} + +static void set_power_scale(unsigned int cpu, unsigned long power) +{ + per_cpu(cpu_scale, cpu) = power; +} + +unsigned long scale_cpu_capacity(struct sched_domain *sd, int cpu) +{ +#ifdef CONFIG_CPU_FREQ + unsigned long max_freq_scale = cpufreq_scale_max_freq_capacity(cpu); + + return per_cpu(cpu_scale, cpu) * max_freq_scale >> SCHED_CAPACITY_SHIFT; +#else + return per_cpu(cpu_scale, cpu); +#endif +} + +static void set_capacity_scale(unsigned int cpu, unsigned long capacity) +{ + per_cpu(cpu_scale, cpu) = capacity; +} + static int __init get_cpu_for_node(struct device_node *node) { struct device_node *cpu_node; @@ -161,6 +203,46 @@ return 0; } +struct cpu_efficiency { + const char *compatible; + unsigned long efficiency; +}; + +/* + * Table of relative efficiency of each processors + * The efficiency value must fit in 20bit and the final + * cpu_scale value must be in the range + * 0 < cpu_scale < 3*SCHED_CAPACITY_SCALE/2 + * in order to return at most 1 when DIV_ROUND_CLOSEST + * is used to compute the capacity of a CPU. + * Processors that are not defined in the table, + * use the default SCHED_CAPACITY_SCALE value for cpu_scale. + */ +static const struct cpu_efficiency table_efficiency[] = { + { NULL, }, +}; + +static unsigned long *__cpu_capacity; +#define cpu_capacity(cpu) __cpu_capacity[cpu] + +static unsigned long middle_capacity = 1; + +static DEFINE_PER_CPU(unsigned long, cpu_efficiency) = SCHED_CAPACITY_SCALE; + +unsigned long arch_get_cpu_efficiency(int cpu) +{ + return per_cpu(cpu_efficiency, cpu); +} +EXPORT_SYMBOL(arch_get_cpu_efficiency); + +/* + * Iterate all CPUs' descriptor in DT and compute the efficiency + * (as per table_efficiency). Also calculate a middle efficiency + * as close as possible to (max{eff_i} - min{eff_i}) / 2 + * This is later used to scale the cpu_power field such that an + * 'average' CPU is of middle power. Also see the comments near + * table_efficiency[] and update_cpu_power(). + */ static int __init parse_dt_topology(void) { struct device_node *cn, *map; @@ -200,17 +282,179 @@ return ret; } +static void __init parse_dt_cpu_power(void) +{ + const struct cpu_efficiency *cpu_eff; + struct device_node *cn; + unsigned long min_capacity = ULONG_MAX; + unsigned long max_capacity = 0; + unsigned long capacity = 0; + int cpu; + + __cpu_capacity = kcalloc(nr_cpu_ids, sizeof(*__cpu_capacity), + GFP_NOWAIT); + + for_each_possible_cpu(cpu) { + const u32 *rate; + int len; + u32 efficiency; + + /* Too early to use cpu->of_node */ + cn = of_get_cpu_node(cpu, NULL); + if (!cn) { + pr_err("Missing device node for CPU %d\n", cpu); + continue; + } + + /* + * The CPU efficiency value passed from the device tree + * overrides the value defined in the table_efficiency[] + */ + if (of_property_read_u32(cn, "efficiency", &efficiency) < 0) { + + for (cpu_eff = table_efficiency; + cpu_eff->compatible; cpu_eff++) + + if (of_device_is_compatible(cn, + cpu_eff->compatible)) + break; + + if (cpu_eff->compatible == NULL) { + pr_warn("%s: Unknown CPU type\n", + cn->full_name); + continue; + } + + efficiency = cpu_eff->efficiency; + } + + per_cpu(cpu_efficiency, cpu) = efficiency; + + rate = of_get_property(cn, "clock-frequency", &len); + if (!rate || len != 4) { + pr_err("%s: Missing clock-frequency property\n", + cn->full_name); + continue; + } + + capacity = ((be32_to_cpup(rate)) >> 20) * efficiency; + + /* Save min capacity of the system */ + if (capacity < min_capacity) + min_capacity = capacity; + + /* Save max capacity of the system */ + if (capacity > max_capacity) + max_capacity = capacity; + + cpu_capacity(cpu) = capacity; + } + + /* If min and max capacities are equal we bypass the update of the + * cpu_scale because all CPUs have the same capacity. Otherwise, we + * compute a middle_capacity factor that will ensure that the capacity + * of an 'average' CPU of the system will be as close as possible to + * SCHED_CAPACITY_SCALE, which is the default value, but with the + * constraint explained near table_efficiency[]. + */ + if (min_capacity == max_capacity) + return; + else if (4 * max_capacity < (3 * (max_capacity + min_capacity))) + middle_capacity = (min_capacity + max_capacity) + >> (SCHED_CAPACITY_SHIFT+1); + else + middle_capacity = ((max_capacity / 3) + >> (SCHED_CAPACITY_SHIFT-1)) + 1; +} + +/* + * Look for a customed capacity of a CPU in the cpu_topo_data table during the + * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the + * function returns directly for SMP system. + */ +static void update_cpu_power(unsigned int cpu) +{ + if (!cpu_capacity(cpu)) + return; + + set_power_scale(cpu, cpu_capacity(cpu) / middle_capacity); + + pr_info("CPU%u: update cpu_power %lu\n", + cpu, arch_scale_freq_power(NULL, cpu)); +} + /* * cpu topology table */ struct cpu_topology cpu_topology[NR_CPUS]; EXPORT_SYMBOL_GPL(cpu_topology); +/* sd energy functions */ +static inline +const struct sched_group_energy * const cpu_cluster_energy(int cpu) +{ + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL1]; + + if (!sge) { + pr_warn("Invalid sched_group_energy for Cluster%d\n", cpu); + return NULL; + } + + return sge; +} + +static inline +const struct sched_group_energy * const cpu_core_energy(int cpu) +{ + struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL0]; + + if (!sge) { + pr_warn("Invalid sched_group_energy for CPU%d\n", cpu); + return NULL; + } + + return sge; +} + const struct cpumask *cpu_coregroup_mask(int cpu) { return &cpu_topology[cpu].core_sibling; } +static int cpu_cpu_flags(void) +{ + return SD_ASYM_CPUCAPACITY; +} + +static inline int cpu_corepower_flags(void) +{ + return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN | \ + SD_SHARE_CAP_STATES; +} + +static struct sched_domain_topology_level arm64_topology[] = { +#ifdef CONFIG_SCHED_MC + { cpu_coregroup_mask, cpu_corepower_flags, cpu_core_energy, SD_INIT_NAME(MC) }, +#endif + { cpu_cpu_mask, cpu_cpu_flags, cpu_cluster_energy, SD_INIT_NAME(DIE) }, + { NULL, }, +}; + +static void update_cpu_capacity(unsigned int cpu) +{ + unsigned long capacity = SCHED_CAPACITY_SCALE; + + if (sched_energy_aware && cpu_core_energy(cpu)) { + int max_cap_idx = cpu_core_energy(cpu)->nr_cap_states - 1; + capacity = cpu_core_energy(cpu)->cap_states[max_cap_idx].cap; + } + + set_capacity_scale(cpu, capacity); + + pr_info("CPU%d: update cpu_capacity %lu\n", + cpu, arch_scale_cpu_capacity(NULL, cpu)); +} + static void update_siblings_masks(unsigned int cpuid) { struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; @@ -272,6 +516,8 @@ topology_populated: update_siblings_masks(cpuid); + update_cpu_power(cpuid); + update_cpu_capacity(cpuid); } static void __init reset_cpu_topology(void) @@ -292,14 +538,33 @@ } } +static void __init reset_cpu_power(void) +{ + unsigned int cpu; + + for_each_possible_cpu(cpu) + set_power_scale(cpu, SCHED_CAPACITY_SCALE); +} + void __init init_cpu_topology(void) { + int cpu; + reset_cpu_topology(); /* * Discard anything that was parsed if we hit an error so we * don't use partial information. */ - if (of_have_populated_dt() && parse_dt_topology()) + if (of_have_populated_dt() && parse_dt_topology()) { reset_cpu_topology(); + } else { + set_sched_topology(arm64_topology); + for_each_possible_cpu(cpu) + update_siblings_masks(cpu); + } + + reset_cpu_power(); + parse_dt_cpu_power(); + init_sched_energy_costs(); } diff -ruw linux-4.4.115/arch/arm64/kernel/traps.c linux-4.4.115-fbx/arch/arm64/kernel/traps.c --- linux-4.4.115/arch/arm64/kernel/traps.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/traps.c 2019-10-29 09:26:23.009196974 +0100 @@ -33,14 +33,20 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include +#include +#include + +#include static const char *handler[]= { "Synchronous Abort", @@ -146,17 +152,32 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) { struct stackframe frame; + unsigned long irq_stack_ptr; + int skip; + + pr_debug("%s(regs = %pK tsk = %pK)\n", __func__, regs, tsk); + + if (!tsk) + tsk = current; + + if (!try_get_task_stack(tsk)) + return; + + /* + * Switching between stacks is valid when tracing current and in + * non-preemptible context. + */ + if (tsk == current && !preemptible()) + irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id()); + else + irq_stack_ptr = 0; pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); if (!tsk) tsk = current; - if (regs) { - frame.fp = regs->regs[29]; - frame.sp = regs->sp; - frame.pc = regs->pc; - } else if (tsk == current) { + if (tsk == current) { frame.fp = (unsigned long)__builtin_frame_address(0); frame.sp = current_stack_pointer; frame.pc = (unsigned long)dump_backtrace; @@ -168,24 +189,54 @@ frame.sp = thread_saved_sp(tsk); frame.pc = thread_saved_pc(tsk); } +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + frame.graph = tsk->curr_ret_stack; +#endif - pr_emerg("Call trace:\n"); + skip = !!regs; + printk("Call trace:\n"); while (1) { unsigned long where = frame.pc; unsigned long stack; int ret; + /* skip until specified stack frame */ + if (!skip) { dump_backtrace_entry(where); - ret = unwind_frame(&frame); + } else if (frame.fp == regs->regs[29]) { + skip = 0; + /* + * Mostly, this is the case where this function is + * called in panic/abort. As exception handler's + * stack frame does not contain the corresponding pc + * at which an exception has taken place, use regs->pc + * instead. + */ + dump_backtrace_entry(regs->pc); + } + ret = unwind_frame(tsk, &frame); if (ret < 0) break; stack = frame.sp; - if (in_exception_text(where)) + if (in_exception_text(where)) { + /* + * If we switched to the irq_stack before calling this + * exception handler, then the pt_regs will be on the + * task stack. The easiest way to tell is if the large + * pt_regs would overlap with the end of the irq_stack. + */ + if (stack < irq_stack_ptr && + (stack + sizeof(struct pt_regs)) > irq_stack_ptr) + stack = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); + dump_mem("", "Exception stack", stack, stack + sizeof(struct pt_regs), false); } } + put_task_stack(tsk); +} + void show_stack(struct task_struct *tsk, unsigned long *sp) { dump_backtrace(NULL, tsk); @@ -199,10 +250,9 @@ #endif #define S_SMP " SMP" -static int __die(const char *str, int err, struct thread_info *thread, - struct pt_regs *regs) +static int __die(const char *str, int err, struct pt_regs *regs) { - struct task_struct *tsk = thread->task; + struct task_struct *tsk = current; static int die_counter; int ret; @@ -217,12 +267,10 @@ print_modules(); __show_regs(regs); pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n", - TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), thread + 1); + TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), + end_of_stack(tsk)); if (!user_mode(regs) || in_interrupt()) { - dump_mem(KERN_EMERG, "Stack: ", regs->sp, - THREAD_SIZE + (unsigned long)task_stack_page(tsk), - compat_user_mode(regs)); dump_backtrace(regs, tsk); dump_instr(KERN_EMERG, regs); } @@ -230,39 +278,75 @@ return ret; } -static DEFINE_RAW_SPINLOCK(die_lock); +static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED; +static int die_owner = -1; +static unsigned int die_nest_count; -/* - * This function is protected against re-entrancy. - */ -void die(const char *str, struct pt_regs *regs, int err) +static unsigned long oops_begin(void) { - struct thread_info *thread = current_thread_info(); - int ret; + int cpu; + unsigned long flags; oops_enter(); - raw_spin_lock_irq(&die_lock); + /* racy, but better than risking deadlock. */ + raw_local_irq_save(flags); + cpu = smp_processor_id(); + if (!arch_spin_trylock(&die_lock)) { + if (cpu == die_owner) + /* nested oops. should stop eventually */; + else + arch_spin_lock(&die_lock); + } + die_nest_count++; + die_owner = cpu; console_verbose(); bust_spinlocks(1); - ret = __die(str, err, thread, regs); + return flags; +} - if (regs && kexec_should_crash(thread->task)) +static void oops_end(unsigned long flags, struct pt_regs *regs, int notify) +{ + if (regs && kexec_should_crash(current)) crash_kexec(regs); bust_spinlocks(0); + die_owner = -1; add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); - raw_spin_unlock_irq(&die_lock); + die_nest_count--; + if (!die_nest_count) + /* Nest count reaches zero, release the lock. */ + arch_spin_unlock(&die_lock); + raw_local_irq_restore(flags); oops_exit(); if (in_interrupt()) panic("Fatal exception in interrupt"); if (panic_on_oops) panic("Fatal exception"); - if (ret != NOTIFY_STOP) + if (notify != NOTIFY_STOP) do_exit(SIGSEGV); } +/* + * This function is protected against re-entrancy. + */ +void die(const char *str, struct pt_regs *regs, int err) +{ + enum bug_trap_type bug_type = BUG_TRAP_TYPE_NONE; + unsigned long flags = oops_begin(); + int ret; + + if (!user_mode(regs)) + bug_type = report_bug(regs->pc, regs); + if (bug_type != BUG_TRAP_TYPE_NONE) + str = "Oops - BUG"; + + ret = __die(str, err, regs); + + oops_end(flags, regs, ret); +} + void arm64_notify_die(const char *str, struct pt_regs *regs, struct siginfo *info, int err) { @@ -350,6 +434,8 @@ if (call_undef_hook(regs) == 0) return; + trace_undef_instr(regs, (void *)pc); + if (unhandled_signal(current, SIGILL) && show_unhandled_signals_ratelimited()) { pr_info("%s[%d]: undefined instruction: pc=%p\n", current->comm, task_pid_nr(current), pc); @@ -364,6 +450,38 @@ arm64_notify_die("Oops - undefined instruction", regs, &info, 0); } +static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) +{ + int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT; + + isb(); + if (rt != 31) + regs->regs[rt] = arch_counter_get_cntvct(); + regs->pc += 4; +} + +static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) +{ + int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT; + + if (rt != 31) + regs->regs[rt] = read_sysreg(cntfrq_el0); + regs->pc += 4; +} + +asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs) +{ + if ((esr & ESR_ELx_SYS64_ISS_SYS_OP_MASK) == ESR_ELx_SYS64_ISS_SYS_CNTVCT) { + cntvct_read_handler(esr, regs); + return; + } else if ((esr & ESR_ELx_SYS64_ISS_SYS_OP_MASK) == ESR_ELx_SYS64_ISS_SYS_CNTFRQ) { + cntfrq_read_handler(esr, regs); + return; + } + + do_undefinstr(regs); +} + long compat_arm_syscall(struct pt_regs *regs); asmlinkage long do_ni_syscall(struct pt_regs *regs) @@ -430,7 +548,7 @@ const char *esr_get_class_string(u32 esr) { - return esr_class_str[esr >> ESR_ELx_EC_SHIFT]; + return esr_class_str[ESR_ELx_EC(esr)]; } /* @@ -444,6 +562,12 @@ pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n", handler[reason], esr, esr_get_class_string(esr)); + if (esr >> ESR_ELx_EC_SHIFT == ESR_ELx_EC_SERROR) { + pr_crit("System error detected. ESR.ISS = %08x\n", + esr & 0xffffff); + arm64_check_cache_ecc(NULL); + } + die("Oops - bad mode", regs, 0); local_irq_disable(); panic("bad mode"); @@ -476,22 +600,22 @@ void __pte_error(const char *file, int line, unsigned long val) { - pr_crit("%s:%d: bad pte %016lx.\n", file, line, val); + pr_err("%s:%d: bad pte %016lx.\n", file, line, val); } void __pmd_error(const char *file, int line, unsigned long val) { - pr_crit("%s:%d: bad pmd %016lx.\n", file, line, val); + pr_err("%s:%d: bad pmd %016lx.\n", file, line, val); } void __pud_error(const char *file, int line, unsigned long val) { - pr_crit("%s:%d: bad pud %016lx.\n", file, line, val); + pr_err("%s:%d: bad pud %016lx.\n", file, line, val); } void __pgd_error(const char *file, int line, unsigned long val) { - pr_crit("%s:%d: bad pgd %016lx.\n", file, line, val); + pr_err("%s:%d: bad pgd %016lx.\n", file, line, val); } /* GENERIC_BUG traps */ diff -ruw linux-4.4.115/arch/arm64/kernel/vdso/gettimeofday.S linux-4.4.115-fbx/arch/arm64/kernel/vdso/gettimeofday.S --- linux-4.4.115/arch/arm64/kernel/vdso/gettimeofday.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/vdso/gettimeofday.S 2019-10-29 09:26:23.009196974 +0100 @@ -26,24 +26,109 @@ #define NSEC_PER_SEC_HI16 0x3b9a vdso_data .req x6 -use_syscall .req w7 -seqcnt .req w8 +seqcnt .req w7 +w_tmp .req w8 +x_tmp .req x8 + +/* + * Conventions for macro arguments: + * - An argument is write-only if its name starts with "res". + * - All other arguments are read-only, unless otherwise specified. + */ .macro seqcnt_acquire 9999: ldr seqcnt, [vdso_data, #VDSO_TB_SEQ_COUNT] tbnz seqcnt, #0, 9999b dmb ishld - ldr use_syscall, [vdso_data, #VDSO_USE_SYSCALL] .endm - .macro seqcnt_read, cnt + .macro seqcnt_check fail dmb ishld - ldr \cnt, [vdso_data, #VDSO_TB_SEQ_COUNT] + ldr w_tmp, [vdso_data, #VDSO_TB_SEQ_COUNT] + cmp w_tmp, seqcnt + b.ne \fail .endm - .macro seqcnt_check, cnt, fail - cmp \cnt, seqcnt - b.ne \fail + .macro syscall_check fail + ldr w_tmp, [vdso_data, #VDSO_USE_SYSCALL] + cbnz w_tmp, \fail + .endm + + .macro get_nsec_per_sec res + mov \res, #NSEC_PER_SEC_LO16 + movk \res, #NSEC_PER_SEC_HI16, lsl #16 + .endm + + /* + * Returns the clock delta, in nanoseconds left-shifted by the clock + * shift. + */ + .macro get_clock_shifted_nsec res, cycle_last, mult + /* Read the virtual counter. */ + isb + mrs x_tmp, cntvct_el0 + /* Calculate cycle delta and convert to ns. */ + sub \res, x_tmp, \cycle_last + /* We can only guarantee 56 bits of precision. */ + movn x_tmp, #0xff00, lsl #48 + and \res, x_tmp, \res + mul \res, \res, \mult + .endm + + /* + * Returns in res_{sec,nsec} the REALTIME timespec, based on the + * "wall time" (xtime) and the clock_mono delta. + */ + .macro get_ts_realtime res_sec, res_nsec, \ + clock_nsec, xtime_sec, xtime_nsec, nsec_to_sec + add \res_nsec, \clock_nsec, \xtime_nsec + udiv x_tmp, \res_nsec, \nsec_to_sec + add \res_sec, \xtime_sec, x_tmp + msub \res_nsec, x_tmp, \nsec_to_sec, \res_nsec + .endm + + /* + * Returns in res_{sec,nsec} the timespec based on the clock_raw delta, + * used for CLOCK_MONOTONIC_RAW. + */ + .macro get_ts_clock_raw res_sec, res_nsec, clock_nsec, nsec_to_sec + udiv \res_sec, \clock_nsec, \nsec_to_sec + msub \res_nsec, \res_sec, \nsec_to_sec, \clock_nsec + .endm + + /* sec and nsec are modified in place. */ + .macro add_ts sec, nsec, ts_sec, ts_nsec, nsec_to_sec + /* Add timespec. */ + add \sec, \sec, \ts_sec + add \nsec, \nsec, \ts_nsec + + /* Normalise the new timespec. */ + cmp \nsec, \nsec_to_sec + b.lt 9999f + sub \nsec, \nsec, \nsec_to_sec + add \sec, \sec, #1 +9999: + cmp \nsec, #0 + b.ge 9998f + add \nsec, \nsec, \nsec_to_sec + sub \sec, \sec, #1 +9998: + .endm + + .macro clock_gettime_return, shift=0 + .if \shift == 1 + lsr x11, x11, x12 + .endif + stp x10, x11, [x1, #TSPEC_TV_SEC] + mov x0, xzr + ret + .endm + + .macro jump_slot jumptable, index, label + .if (. - \jumptable) != 4 * (\index) + .error "Jump slot index mismatch" + .endif + b \label .endm .text @@ -51,18 +136,25 @@ /* int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz); */ ENTRY(__kernel_gettimeofday) .cfi_startproc - mov x2, x30 - .cfi_register x30, x2 - - /* Acquire the sequence counter and get the timespec. */ adr vdso_data, _vdso_data -1: seqcnt_acquire - cbnz use_syscall, 4f - /* If tv is NULL, skip to the timezone code. */ cbz x0, 2f - bl __do_get_tspec - seqcnt_check w9, 1b + + /* Compute the time of day. */ +1: seqcnt_acquire + syscall_check fail=4f + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_mono_mult, w12 = cs_shift */ + ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] + ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] + seqcnt_check fail=1b + + get_nsec_per_sec res=x9 + lsl x9, x9, x12 + + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_realtime res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 /* Convert ns to us. */ mov x13, #1000 @@ -76,95 +168,126 @@ stp w4, w5, [x1, #TZ_MINWEST] 3: mov x0, xzr - ret x2 + ret 4: /* Syscall fallback. */ mov x8, #__NR_gettimeofday svc #0 - ret x2 + ret .cfi_endproc ENDPROC(__kernel_gettimeofday) +#define JUMPSLOT_MAX CLOCK_MONOTONIC_COARSE + /* int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp); */ ENTRY(__kernel_clock_gettime) .cfi_startproc - cmp w0, #CLOCK_REALTIME - ccmp w0, #CLOCK_MONOTONIC, #0x4, ne - b.ne 2f - - mov x2, x30 - .cfi_register x30, x2 - - /* Get kernel timespec. */ + cmp w0, #JUMPSLOT_MAX + b.hi syscall adr vdso_data, _vdso_data -1: seqcnt_acquire - cbnz use_syscall, 7f - - bl __do_get_tspec - seqcnt_check w9, 1b - - mov x30, x2 - - cmp w0, #CLOCK_MONOTONIC - b.ne 6f + adr x_tmp, jumptable + add x_tmp, x_tmp, w0, uxtw #2 + br x_tmp + + ALIGN +jumptable: + jump_slot jumptable, CLOCK_REALTIME, realtime + jump_slot jumptable, CLOCK_MONOTONIC, monotonic + b syscall + b syscall + jump_slot jumptable, CLOCK_MONOTONIC_RAW, monotonic_raw + jump_slot jumptable, CLOCK_REALTIME_COARSE, realtime_coarse + jump_slot jumptable, CLOCK_MONOTONIC_COARSE, monotonic_coarse + + .if (. - jumptable) != 4 * (JUMPSLOT_MAX + 1) + .error "Wrong jumptable size" + .endif + + ALIGN +realtime: + seqcnt_acquire + syscall_check fail=syscall + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_mono_mult, w12 = cs_shift */ + ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] + ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] + seqcnt_check fail=realtime - /* Get wtm timespec. */ - ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC] + /* All computations are done with left-shifted nsecs. */ + get_nsec_per_sec res=x9 + lsl x9, x9, x12 + + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_realtime res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 + clock_gettime_return, shift=1 + + ALIGN +monotonic: + seqcnt_acquire + syscall_check fail=syscall + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_mono_mult, w12 = cs_shift */ + ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] + ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] + ldp x3, x4, [vdso_data, #VDSO_WTM_CLK_SEC] + seqcnt_check fail=monotonic - /* Check the sequence counter. */ - seqcnt_read w9 - seqcnt_check w9, 1b - b 4f -2: - cmp w0, #CLOCK_REALTIME_COARSE - ccmp w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne - b.ne 8f + /* All computations are done with left-shifted nsecs. */ + lsl x4, x4, x12 + get_nsec_per_sec res=x9 + lsl x9, x9, x12 + + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_realtime res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 + + add_ts sec=x10, nsec=x11, ts_sec=x3, ts_nsec=x4, nsec_to_sec=x9 + clock_gettime_return, shift=1 + + ALIGN +monotonic_raw: + seqcnt_acquire + syscall_check fail=syscall + ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] + /* w11 = cs_raw_mult, w12 = cs_shift */ + ldp w12, w11, [vdso_data, #VDSO_CS_SHIFT] + ldp x13, x14, [vdso_data, #VDSO_RAW_TIME_SEC] + seqcnt_check fail=monotonic_raw - /* xtime_coarse_nsec is already right-shifted */ - mov x12, #0 + /* All computations are done with left-shifted nsecs. */ + lsl x14, x14, x12 + get_nsec_per_sec res=x9 + lsl x9, x9, x12 - /* Get coarse timespec. */ - adr vdso_data, _vdso_data -3: seqcnt_acquire + get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 + get_ts_clock_raw res_sec=x10, res_nsec=x11, \ + clock_nsec=x15, nsec_to_sec=x9 + + add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 + clock_gettime_return, shift=1 + + ALIGN +realtime_coarse: + seqcnt_acquire ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] + seqcnt_check fail=realtime_coarse + clock_gettime_return - /* Get wtm timespec. */ + ALIGN +monotonic_coarse: + seqcnt_acquire + ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC] + seqcnt_check fail=monotonic_coarse - /* Check the sequence counter. */ - seqcnt_read w9 - seqcnt_check w9, 3b - - cmp w0, #CLOCK_MONOTONIC_COARSE - b.ne 6f -4: - /* Add on wtm timespec. */ - add x10, x10, x13 - lsl x14, x14, x12 - add x11, x11, x14 + /* Computations are done in (non-shifted) nsecs. */ + get_nsec_per_sec res=x9 + add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 + clock_gettime_return - /* Normalise the new timespec. */ - mov x15, #NSEC_PER_SEC_LO16 - movk x15, #NSEC_PER_SEC_HI16, lsl #16 - lsl x15, x15, x12 - cmp x11, x15 - b.lt 5f - sub x11, x11, x15 - add x10, x10, #1 -5: - cmp x11, #0 - b.ge 6f - add x11, x11, x15 - sub x10, x10, #1 - -6: /* Store to the user timespec. */ - lsr x11, x11, x12 - stp x10, x11, [x1, #TSPEC_TV_SEC] - mov x0, xzr - ret -7: - mov x30, x2 -8: /* Syscall fallback. */ + ALIGN +syscall: /* Syscall fallback. */ mov x8, #__NR_clock_gettime svc #0 ret @@ -176,6 +299,7 @@ .cfi_startproc cmp w0, #CLOCK_REALTIME ccmp w0, #CLOCK_MONOTONIC, #0x4, ne + ccmp w0, #CLOCK_MONOTONIC_RAW, #0x4, ne b.ne 1f ldr x2, 5f @@ -186,7 +310,7 @@ b.ne 4f ldr x2, 6f 2: - cbz w1, 3f + cbz x1, 3f stp xzr, x2, [x1] 3: /* res == NULL. */ @@ -203,46 +327,3 @@ .quad CLOCK_COARSE_RES .cfi_endproc ENDPROC(__kernel_clock_getres) - -/* - * Read the current time from the architected counter. - * Expects vdso_data to be initialised. - * Clobbers the temporary registers (x9 - x15). - * Returns: - * - w9 = vDSO sequence counter - * - (x10, x11) = (ts->tv_sec, shifted ts->tv_nsec) - * - w12 = cs_shift - */ -ENTRY(__do_get_tspec) - .cfi_startproc - - /* Read from the vDSO data page. */ - ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] - ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] - ldp w11, w12, [vdso_data, #VDSO_CS_MULT] - seqcnt_read w9 - - /* Read the virtual counter. */ - isb - mrs x15, cntvct_el0 - - /* Calculate cycle delta and convert to ns. */ - sub x10, x15, x10 - /* We can only guarantee 56 bits of precision. */ - movn x15, #0xff00, lsl #48 - and x10, x15, x10 - mul x10, x10, x11 - - /* Use the kernel time to calculate the new timespec. */ - mov x11, #NSEC_PER_SEC_LO16 - movk x11, #NSEC_PER_SEC_HI16, lsl #16 - lsl x11, x11, x12 - add x15, x10, x14 - udiv x14, x15, x11 - add x10, x13, x14 - mul x13, x14, x11 - sub x11, x15, x13 - - ret - .cfi_endproc -ENDPROC(__do_get_tspec) diff -ruw linux-4.4.115/arch/arm64/kernel/vdso/Makefile linux-4.4.115-fbx/arch/arm64/kernel/vdso/Makefile --- linux-4.4.115/arch/arm64/kernel/vdso/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/vdso/Makefile 2019-01-22 16:16:21.555228766 +0100 @@ -23,7 +23,7 @@ ccflags-y += -Wl,-shared obj-y += vdso.o -extra-y += vdso.lds vdso-offsets.h +extra-y += vdso.lds CPPFLAGS_vdso.lds += -P -C -U$(ARCH) # Force dependency (incbin is bad) @@ -42,11 +42,10 @@ gen-vdsosym := $(srctree)/$(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ define cmd_vdsosym - $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ && \ - cp $@ include/generated/ + $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ endef -$(obj)/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE +include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE $(call if_changed,vdsosym) # Assembly rules for the .S files diff -ruw linux-4.4.115/arch/arm64/kernel/vdso/vdso.S linux-4.4.115-fbx/arch/arm64/kernel/vdso/vdso.S --- linux-4.4.115/arch/arm64/kernel/vdso/vdso.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/vdso/vdso.S 2019-01-22 16:16:21.555228766 +0100 @@ -21,9 +21,8 @@ #include #include - __PAGE_ALIGNED_DATA - .globl vdso_start, vdso_end + .section .rodata .balign PAGE_SIZE vdso_start: .incbin "arch/arm64/kernel/vdso/vdso.so" diff -ruw linux-4.4.115/arch/arm64/kernel/vdso.c linux-4.4.115-fbx/arch/arm64/kernel/vdso.c --- linux-4.4.115/arch/arm64/kernel/vdso.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/vdso.c 2019-10-29 09:26:23.009196974 +0100 @@ -55,7 +55,7 @@ */ static struct page *vectors_page[1]; -static int alloc_vectors_page(void) +static int __init alloc_vectors_page(void) { extern char __kuser_helper_start[], __kuser_helper_end[]; extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[]; @@ -88,7 +88,7 @@ { struct mm_struct *mm = current->mm; unsigned long addr = AARCH32_VECTORS_BASE; - static struct vm_special_mapping spec = { + static const struct vm_special_mapping spec = { .name = "[vectors]", .pages = vectors_page, @@ -114,6 +114,7 @@ static int __init vdso_init(void) { int i; + unsigned long pfn; if (memcmp(&vdso_start, "\177ELF", 4)) { pr_err("vDSO is not a valid ELF object!\n"); @@ -131,11 +132,14 @@ return -ENOMEM; /* Grab the vDSO data page. */ - vdso_pagelist[0] = virt_to_page(vdso_data); + vdso_pagelist[0] = phys_to_page(__pa_symbol(vdso_data)); + /* Grab the vDSO code pages. */ + pfn = sym_to_pfn(&vdso_start); + for (i = 0; i < vdso_pages; i++) - vdso_pagelist[i + 1] = virt_to_page(&vdso_start + i * PAGE_SIZE); + vdso_pagelist[i + 1] = pfn_to_page(pfn + i); /* Populate the special mapping structures */ vdso_spec[0] = (struct vm_special_mapping) { @@ -212,10 +216,16 @@ vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; if (!use_syscall) { + /* tkr_mono.cycle_last == tkr_raw.cycle_last */ vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; + vdso_data->raw_time_sec = tk->raw_sec; + vdso_data->raw_time_nsec = tk->tkr_raw.xtime_nsec; vdso_data->xtime_clock_sec = tk->xtime_sec; vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec; - vdso_data->cs_mult = tk->tkr_mono.mult; + /* tkr_raw.xtime_nsec == 0 */ + vdso_data->cs_mono_mult = tk->tkr_mono.mult; + vdso_data->cs_raw_mult = tk->tkr_raw.mult; + /* tkr_mono.shift == tkr_raw.shift */ vdso_data->cs_shift = tk->tkr_mono.shift; } diff -ruw linux-4.4.115/arch/arm64/kernel/vmlinux.lds.S linux-4.4.115-fbx/arch/arm64/kernel/vmlinux.lds.S --- linux-4.4.115/arch/arm64/kernel/vmlinux.lds.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kernel/vmlinux.lds.S 2019-10-29 09:26:23.009196974 +0100 @@ -11,6 +11,7 @@ #include #include #include +#include #include "image.h" @@ -46,6 +47,27 @@ *(.idmap.text) \ VMLINUX_SYMBOL(__idmap_text_end) = .; +#ifdef CONFIG_HIBERNATION +#define HIBERNATE_TEXT \ + . = ALIGN(SZ_4K); \ + VMLINUX_SYMBOL(__hibernate_exit_text_start) = .;\ + *(.hibernate_exit.text) \ + VMLINUX_SYMBOL(__hibernate_exit_text_end) = .; +#else +#define HIBERNATE_TEXT +#endif + +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +#define TRAMP_TEXT \ + . = ALIGN(PAGE_SIZE); \ + VMLINUX_SYMBOL(__entry_tramp_text_start) = .; \ + *(.entry.tramp.text) \ + . = ALIGN(PAGE_SIZE); \ + VMLINUX_SYMBOL(__entry_tramp_text_end) = .; +#else +#define TRAMP_TEXT +#endif + /* * The size of the PE/COFF section that covers the kernel image, which * runs from stext to _edata, must be a round multiple of the PE/COFF @@ -63,14 +85,19 @@ #endif #if defined(CONFIG_DEBUG_ALIGN_RODATA) -#define ALIGN_DEBUG_RO . = ALIGN(1<phys conversions will fail. */ -ASSERT(_text == (PAGE_OFFSET + TEXT_OFFSET), "HEAD is misaligned") +ASSERT(_text == (KIMAGE_VADDR + TEXT_OFFSET), "HEAD is misaligned") diff -ruw linux-4.4.115/arch/arm64/kvm/Makefile linux-4.4.115-fbx/arch/arm64/kvm/Makefile --- linux-4.4.115/arch/arm64/kvm/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/kvm/Makefile 2019-01-22 16:16:21.555228766 +0100 @@ -10,6 +10,7 @@ ARM=../../../arch/arm/kvm obj-$(CONFIG_KVM_ARM_HOST) += kvm.o +obj-$(CONFIG_KVM_ARM_HOST) += hyp/ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o $(KVM)/vfio.o kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o @@ -22,8 +23,6 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v2-switch.o kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o diff -ruw linux-4.4.115/arch/arm64/lib/clear_user.S linux-4.4.115-fbx/arch/arm64/lib/clear_user.S --- linux-4.4.115/arch/arm64/lib/clear_user.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/lib/clear_user.S 2019-01-22 16:16:21.559228802 +0100 @@ -17,10 +17,7 @@ */ #include -#include -#include -#include -#include +#include .text @@ -33,29 +30,27 @@ * Alignment fixed up by hardware. */ ENTRY(__clear_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) + uaccess_enable_not_uao x2, x3, x4 mov x2, x1 // save the size for fixup return subs x1, x1, #8 b.mi 2f 1: -USER(9f, str xzr, [x0], #8 ) +uao_user_alternative 9f, str, sttr, xzr, x0, 8 subs x1, x1, #8 b.pl 1b 2: adds x1, x1, #4 b.mi 3f -USER(9f, str wzr, [x0], #4 ) +uao_user_alternative 9f, str, sttr, wzr, x0, 4 sub x1, x1, #4 3: adds x1, x1, #2 b.mi 4f -USER(9f, strh wzr, [x0], #2 ) +uao_user_alternative 9f, strh, sttrh, wzr, x0, 2 sub x1, x1, #2 4: adds x1, x1, #1 b.mi 5f -USER(9f, strb wzr, [x0] ) +uao_user_alternative 9f, strb, sttrb, wzr, x0, 0 5: mov x0, #0 -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x2, x3 ret ENDPROC(__clear_user) diff -ruw linux-4.4.115/arch/arm64/lib/copy_from_user.S linux-4.4.115-fbx/arch/arm64/lib/copy_from_user.S --- linux-4.4.115/arch/arm64/lib/copy_from_user.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/lib/copy_from_user.S 2019-01-22 16:16:21.559228802 +0100 @@ -16,11 +16,8 @@ #include -#include -#include #include -#include -#include +#include /* * Copy from user space to a kernel buffer (alignment handled by the hardware) @@ -34,7 +31,7 @@ */ .macro ldrb1 ptr, regB, val - USER(9998f, ldrb \ptr, [\regB], \val) + uao_user_alternative 9998f, ldrb, ldtrb, \ptr, \regB, \val .endm .macro strb1 ptr, regB, val @@ -42,7 +39,7 @@ .endm .macro ldrh1 ptr, regB, val - USER(9998f, ldrh \ptr, [\regB], \val) + uao_user_alternative 9998f, ldrh, ldtrh, \ptr, \regB, \val .endm .macro strh1 ptr, regB, val @@ -50,7 +47,7 @@ .endm .macro ldr1 ptr, regB, val - USER(9998f, ldr \ptr, [\regB], \val) + uao_user_alternative 9998f, ldr, ldtr, \ptr, \regB, \val .endm .macro str1 ptr, regB, val @@ -58,7 +55,7 @@ .endm .macro ldp1 ptr, regB, regC, val - USER(9998f, ldp \ptr, \regB, [\regC], \val) + uao_ldp 9998f, \ptr, \regB, \regC, \val .endm .macro stp1 ptr, regB, regC, val @@ -66,16 +63,14 @@ .endm end .req x5 -ENTRY(__copy_from_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) +ENTRY(__arch_copy_from_user) + uaccess_enable_not_uao x3, x4, x5 add end, x0, x2 #include "copy_template.S" -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x3, x4 mov x0, #0 // Nothing to copy ret -ENDPROC(__copy_from_user) +ENDPROC(__arch_copy_from_user) .section .fixup,"ax" .align 2 diff -ruw linux-4.4.115/arch/arm64/lib/copy_in_user.S linux-4.4.115-fbx/arch/arm64/lib/copy_in_user.S --- linux-4.4.115/arch/arm64/lib/copy_in_user.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/lib/copy_in_user.S 2019-01-22 16:16:21.559228802 +0100 @@ -18,11 +18,8 @@ #include -#include -#include #include -#include -#include +#include /* * Copy from user space to user space (alignment handled by the hardware) @@ -35,45 +32,43 @@ * x0 - bytes not copied */ .macro ldrb1 ptr, regB, val - USER(9998f, ldrb \ptr, [\regB], \val) + uao_user_alternative 9998f, ldrb, ldtrb, \ptr, \regB, \val .endm .macro strb1 ptr, regB, val - USER(9998f, strb \ptr, [\regB], \val) + uao_user_alternative 9998f, strb, sttrb, \ptr, \regB, \val .endm .macro ldrh1 ptr, regB, val - USER(9998f, ldrh \ptr, [\regB], \val) + uao_user_alternative 9998f, ldrh, ldtrh, \ptr, \regB, \val .endm .macro strh1 ptr, regB, val - USER(9998f, strh \ptr, [\regB], \val) + uao_user_alternative 9998f, strh, sttrh, \ptr, \regB, \val .endm .macro ldr1 ptr, regB, val - USER(9998f, ldr \ptr, [\regB], \val) + uao_user_alternative 9998f, ldr, ldtr, \ptr, \regB, \val .endm .macro str1 ptr, regB, val - USER(9998f, str \ptr, [\regB], \val) + uao_user_alternative 9998f, str, sttr, \ptr, \regB, \val .endm .macro ldp1 ptr, regB, regC, val - USER(9998f, ldp \ptr, \regB, [\regC], \val) + uao_ldp 9998f, \ptr, \regB, \regC, \val .endm .macro stp1 ptr, regB, regC, val - USER(9998f, stp \ptr, \regB, [\regC], \val) + uao_stp 9998f, \ptr, \regB, \regC, \val .endm end .req x5 ENTRY(__copy_in_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) + uaccess_enable_not_uao x3, x4, x5 add end, x0, x2 #include "copy_template.S" -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x3, x4 mov x0, #0 ret ENDPROC(__copy_in_user) diff -ruw linux-4.4.115/arch/arm64/lib/copy_page.S linux-4.4.115-fbx/arch/arm64/lib/copy_page.S --- linux-4.4.115/arch/arm64/lib/copy_page.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/lib/copy_page.S 2019-01-22 16:16:21.559228802 +0100 @@ -18,6 +18,8 @@ #include #include #include +#include +#include /* * Copy a page from src to dest (both are page aligned) @@ -27,20 +29,65 @@ * x1 - src */ ENTRY(copy_page) - /* Assume cache line size is 64 bytes. */ - prfm pldl1strm, [x1, #64] -1: ldp x2, x3, [x1] +alternative_if_not ARM64_HAS_NO_HW_PREFETCH + nop + nop +alternative_else + # Prefetch two cache lines ahead. + prfm pldl1strm, [x1, #128] + prfm pldl1strm, [x1, #256] +alternative_endif + + ldp x2, x3, [x1] ldp x4, x5, [x1, #16] ldp x6, x7, [x1, #32] ldp x8, x9, [x1, #48] - add x1, x1, #64 - prfm pldl1strm, [x1, #64] + ldp x10, x11, [x1, #64] + ldp x12, x13, [x1, #80] + ldp x14, x15, [x1, #96] + ldp x16, x17, [x1, #112] + + mov x18, #(PAGE_SIZE - 128) + add x1, x1, #128 +1: + subs x18, x18, #128 + +alternative_if_not ARM64_HAS_NO_HW_PREFETCH + nop +alternative_else + prfm pldl1strm, [x1, #384] +alternative_endif + stnp x2, x3, [x0] + ldp x2, x3, [x1] stnp x4, x5, [x0, #16] + ldp x4, x5, [x1, #16] stnp x6, x7, [x0, #32] + ldp x6, x7, [x1, #32] stnp x8, x9, [x0, #48] - add x0, x0, #64 - tst x1, #(PAGE_SIZE - 1) - b.ne 1b + ldp x8, x9, [x1, #48] + stnp x10, x11, [x0, #64] + ldp x10, x11, [x1, #64] + stnp x12, x13, [x0, #80] + ldp x12, x13, [x1, #80] + stnp x14, x15, [x0, #96] + ldp x14, x15, [x1, #96] + stnp x16, x17, [x0, #112] + ldp x16, x17, [x1, #112] + + add x0, x0, #128 + add x1, x1, #128 + + b.gt 1b + + stnp x2, x3, [x0] + stnp x4, x5, [x0, #16] + stnp x6, x7, [x0, #32] + stnp x8, x9, [x0, #48] + stnp x10, x11, [x0, #64] + stnp x12, x13, [x0, #80] + stnp x14, x15, [x0, #96] + stnp x16, x17, [x0, #112] + ret ENDPROC(copy_page) diff -ruw linux-4.4.115/arch/arm64/lib/copy_to_user.S linux-4.4.115-fbx/arch/arm64/lib/copy_to_user.S --- linux-4.4.115/arch/arm64/lib/copy_to_user.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/lib/copy_to_user.S 2019-01-22 16:16:21.559228802 +0100 @@ -16,11 +16,8 @@ #include -#include -#include #include -#include -#include +#include /* * Copy to user space from a kernel buffer (alignment handled by the hardware) @@ -37,7 +34,7 @@ .endm .macro strb1 ptr, regB, val - USER(9998f, strb \ptr, [\regB], \val) + uao_user_alternative 9998f, strb, sttrb, \ptr, \regB, \val .endm .macro ldrh1 ptr, regB, val @@ -45,7 +42,7 @@ .endm .macro strh1 ptr, regB, val - USER(9998f, strh \ptr, [\regB], \val) + uao_user_alternative 9998f, strh, sttrh, \ptr, \regB, \val .endm .macro ldr1 ptr, regB, val @@ -53,7 +50,7 @@ .endm .macro str1 ptr, regB, val - USER(9998f, str \ptr, [\regB], \val) + uao_user_alternative 9998f, str, sttr, \ptr, \regB, \val .endm .macro ldp1 ptr, regB, regC, val @@ -61,20 +58,18 @@ .endm .macro stp1 ptr, regB, regC, val - USER(9998f, stp \ptr, \regB, [\regC], \val) + uao_stp 9998f, \ptr, \regB, \regC, \val .endm end .req x5 -ENTRY(__copy_to_user) -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) +ENTRY(__arch_copy_to_user) + uaccess_enable_not_uao x3, x4, x5 add end, x0, x2 #include "copy_template.S" -ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ - CONFIG_ARM64_PAN) + uaccess_disable_not_uao x3, x4 mov x0, #0 ret -ENDPROC(__copy_to_user) +ENDPROC(__arch_copy_to_user) .section .fixup,"ax" .align 2 diff -ruw linux-4.4.115/arch/arm64/lib/Makefile linux-4.4.115-fbx/arch/arm64/lib/Makefile --- linux-4.4.115/arch/arm64/lib/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/lib/Makefile 2019-01-22 16:16:21.559228802 +0100 @@ -4,15 +4,16 @@ memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ strchr.o strrchr.o -# Tell the compiler to treat all general purpose registers as -# callee-saved, which allows for efficient runtime patching of the bl -# instruction in the caller with an atomic instruction when supported by -# the CPU. Result and argument registers are handled correctly, based on -# the function prototype. +# Tell the compiler to treat all general purpose registers (with the +# exception of the IP registers, which are already handled by the caller +# in case of a PLT) as callee-saved, which allows for efficient runtime +# patching of the bl instruction in the caller with an atomic instruction +# when supported by the CPU. Result and argument registers are handled +# correctly, based on the function prototype. lib-$(CONFIG_ARM64_LSE_ATOMICS) += atomic_ll_sc.o CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ -ffixed-x3 -ffixed-x4 -ffixed-x5 -ffixed-x6 \ -ffixed-x7 -fcall-saved-x8 -fcall-saved-x9 \ -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \ -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \ - -fcall-saved-x16 -fcall-saved-x17 -fcall-saved-x18 + -fcall-saved-x18 diff -ruw linux-4.4.115/arch/arm64/Makefile linux-4.4.115-fbx/arch/arm64/Makefile --- linux-4.4.115/arch/arm64/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/Makefile 2019-10-29 09:26:22.989196778 +0100 @@ -15,6 +15,10 @@ OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S GZFLAGS :=-9 +ifneq ($(CONFIG_RELOCATABLE),) +LDFLAGS_vmlinux += -pie -Bsymbolic +endif + KBUILD_DEFCONFIG := defconfig # Check for binutils support for specific extensions @@ -26,8 +30,17 @@ endif endif -KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) +ifeq ($(cc-name),clang) +# This is a workaround for https://bugs.llvm.org/show_bug.cgi?id=30792. +# TODO: revert when this is fixed in LLVM. +KBUILD_CFLAGS += -mno-implicit-float +else +KBUILD_CFLAGS += -mgeneral-regs-only +endif +KBUILD_CFLAGS += $(lseinstr) +KBUILD_CFLAGS += -fno-pic KBUILD_CFLAGS += $(call cc-option, -mpc-relative-literal-loads) +KBUILD_CFLAGS += -fno-asynchronous-unwind-tables KBUILD_AFLAGS += $(lseinstr) ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) @@ -42,10 +55,14 @@ CHECKFLAGS += -D__aarch64__ -ifeq ($(CONFIG_ARM64_ERRATUM_843419), y) +ifeq ($(CONFIG_ARM64_MODULE_CMODEL_LARGE), y) KBUILD_CFLAGS_MODULE += -mcmodel=large endif +ifeq ($(CONFIG_ARM64_MODULE_PLTS),y) +KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/arm64/kernel/module.lds +endif + # Default value head-y := arch/arm64/kernel/head.o @@ -56,6 +73,10 @@ TEXT_OFFSET := 0x00080000 endif +ifeq ($(cc-name),clang) +KBUILD_CFLAGS += $(call cc-disable-warning, asm-operand-widths) +endif + # KASAN_SHADOW_OFFSET = VA_START + (1 << (VA_BITS - 3)) - (1 << 61) # in 32-bit arithmetic KASAN_SHADOW_OFFSET := $(shell printf "0x%08x00000000\n" $$(( \ @@ -74,9 +95,18 @@ core-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a # Default target when executing plain make +ifeq ($(CONFIG_BUILD_ARM64_APPENDED_DTB_IMAGE),y) +KBUILD_IMAGE := $(subst $\",,$(CONFIG_BUILD_ARM64_APPENDED_KERNEL_IMAGE_NAME)) +else KBUILD_IMAGE := Image.gz +endif + KBUILD_DTBS := dtbs +ifeq ($(CONFIG_BUILD_ARM64_DT_OVERLAY),y) +export DTC_FLAGS := -@ +endif + all: $(KBUILD_IMAGE) $(KBUILD_DTBS) boot := arch/arm64/boot @@ -101,6 +131,9 @@ dtbs_install: $(Q)$(MAKE) $(dtbinst)=$(boot)/dts +Image-dtb Image.gz-dtb: vmlinux scripts dtbs + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + PHONY += vdso_install vdso_install: $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@ @@ -110,6 +143,16 @@ $(Q)$(MAKE) $(clean)=$(boot) $(Q)$(MAKE) $(clean)=$(boot)/dts +# We need to generate vdso-offsets.h before compiling certain files in kernel/. +# In order to do that, we should use the archprepare target, but we can't since +# asm-offsets.h is included in some files used to generate vdso-offsets.h, and +# asm-offsets.h is built in prepare0, for which archprepare is a dependency. +# Therefore we need to generate the header after prepare0 has been made, hence +# this hack. +prepare: vdso_prepare +vdso_prepare: prepare0 + $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso include/generated/vdso-offsets.h + define archhelp echo '* Image.gz - Compressed kernel image (arch/$(ARCH)/boot/Image.gz)' echo ' Image - Uncompressed kernel image (arch/$(ARCH)/boot/Image)' diff -ruw linux-4.4.115/arch/arm64/mm/cache.S linux-4.4.115-fbx/arch/arm64/mm/cache.S --- linux-4.4.115/arch/arm64/mm/cache.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/cache.S 2019-01-22 16:16:21.559228802 +0100 @@ -23,8 +23,80 @@ #include #include #include +#include -#include "proc-macros.S" +/* + * __flush_dcache_all() + * + * Flush the whole D-cache. + * + * Corrupted registers: x0-x7, x9-x11 + */ +__flush_dcache_all: + dmb sy // ensure ordering with previous memory accesses + mrs x0, clidr_el1 // read clidr + and x3, x0, #0x7000000 // extract loc from clidr + lsr x3, x3, #23 // left align loc bit field + cbz x3, finished // if loc is 0, then no need to clean + mov x10, #0 // start clean at cache level 0 +loop1: + add x2, x10, x10, lsr #1 // work out 3x current cache level + lsr x1, x0, x2 // extract cache type bits from clidr + and x1, x1, #7 // mask of the bits for current cache only + cmp x1, #2 // see what cache we have at this level + b.lt skip // skip if no cache, or just i-cache + save_and_disable_irqs x9 // make CSSELR and CCSIDR access atomic + msr csselr_el1, x10 // select current cache level in csselr + isb // isb to sych the new cssr&csidr + mrs x1, ccsidr_el1 // read the new ccsidr + restore_irqs x9 + and x2, x1, #7 // extract the length of the cache lines + add x2, x2, #4 // add 4 (line length offset) + mov x4, #0x3ff + and x4, x4, x1, lsr #3 // find maximum number on the way size + clz w5, w4 // find bit position of way size increment + mov x7, #0x7fff + and x7, x7, x1, lsr #13 // extract max number of the index size +loop2: + mov x9, x4 // create working copy of max way size +loop3: + lsl x6, x9, x5 + orr x11, x10, x6 // factor way and cache number into x11 + lsl x6, x7, x2 + orr x11, x11, x6 // factor index number into x11 + dc cisw, x11 // clean & invalidate by set/way + subs x9, x9, #1 // decrement the way + b.ge loop3 + subs x7, x7, #1 // decrement the index + b.ge loop2 +skip: + add x10, x10, #2 // increment cache number + cmp x3, x10 + b.gt loop1 +finished: + mov x10, #0 // swith back to cache level 0 + msr csselr_el1, x10 // select current cache level in csselr + dsb sy + isb + ret +ENDPROC(__flush_dcache_all) + +/* + * flush_cache_all() + * + * Flush the entire cache system. The data cache flush is now achieved + * using atomic clean / invalidates working outwards from L1 cache. This + * is done using Set/Way based cache maintainance instructions. The + * instruction cache can still be invalidated back to the point of + * unification in a single instruction. + */ +ENTRY(flush_cache_all) + mov x12, lr + bl __flush_dcache_all + mov x0, #0 + ic ialluis // I+BTB cache invalidate + ret x12 +ENDPROC(flush_cache_all) /* * flush_icache_range(start,end) @@ -50,11 +122,12 @@ * - end - virtual end address of region */ ENTRY(__flush_cache_user_range) + uaccess_ttbr0_enable x2, x3, x4 dcache_line_size x2, x3 sub x3, x2, #1 bic x4, x0, x3 1: -USER(9f, dc cvau, x4 ) // clean D line to PoU +user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE add x4, x4, x2 cmp x4, x1 b.lo 1b @@ -71,36 +144,44 @@ dsb ish isb mov x0, #0 +1: + uaccess_ttbr0_disable x1, x2 ret 9: mov x0, #-EFAULT - ret + b 1b ENDPROC(flush_icache_range) ENDPROC(__flush_cache_user_range) /* * __flush_dcache_area(kaddr, size) * - * Ensure that the data held in the page kaddr is written back to the - * page in question. + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * are cleaned and invalidated to the PoC. * * - kaddr - kernel address * - size - size in question */ ENTRY(__flush_dcache_area) - dcache_line_size x2, x3 - add x1, x0, x1 - sub x3, x2, #1 - bic x0, x0, x3 -1: dc civac, x0 // clean & invalidate D line / unified line - add x0, x0, x2 - cmp x0, x1 - b.lo 1b - dsb sy + dcache_by_line_op civac, sy, x0, x1, x2, x3 ret ENDPIPROC(__flush_dcache_area) /* + * __clean_dcache_area_pou(kaddr, size) + * + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * are cleaned to the PoU. + * + * - kaddr - kernel address + * - size - size in question + */ +ENTRY(__clean_dcache_area_pou) + dcache_by_line_op cvau, ish, x0, x1, x2, x3 + ret +ENDPROC(__clean_dcache_area_pou) + +/* * __inval_cache_range(start, end) * - start - start address of region * - end - end address of region @@ -113,7 +194,7 @@ * - start - virtual start address of region * - end - virtual end address of region */ -__dma_inv_range: +ENTRY(__dma_inv_range) dcache_line_size x2, x3 sub x3, x2, #1 tst x1, x3 // end cache line aligned? @@ -139,7 +220,7 @@ * - start - virtual start address of region * - end - virtual end address of region */ -__dma_clean_range: +ENTRY(__dma_clean_range) dcache_line_size x2, x3 sub x3, x2, #1 bic x0, x0, x3 diff -ruw linux-4.4.115/arch/arm64/mm/context.c linux-4.4.115-fbx/arch/arm64/mm/context.c --- linux-4.4.115/arch/arm64/mm/context.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/context.c 2019-01-22 16:16:21.559228802 +0100 @@ -38,7 +38,16 @@ #define ASID_MASK (~GENMASK(asid_bits - 1, 0)) #define ASID_FIRST_VERSION (1UL << asid_bits) -#define NUM_USER_ASIDS ASID_FIRST_VERSION + +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +#define NUM_USER_ASIDS (ASID_FIRST_VERSION >> 1) +#define asid2idx(asid) (((asid) & ~ASID_MASK) >> 1) +#define idx2asid(idx) (((idx) << 1) & ~ASID_MASK) +#else +#define NUM_USER_ASIDS (ASID_FIRST_VERSION) +#define asid2idx(asid) ((asid) & ~ASID_MASK) +#define idx2asid(idx) asid2idx(idx) +#endif static void flush_context(unsigned int cpu) { @@ -65,7 +74,7 @@ */ if (asid == 0) asid = per_cpu(reserved_asids, i); - __set_bit(asid & ~ASID_MASK, asid_map); + __set_bit(asid2idx(asid), asid_map); per_cpu(reserved_asids, i) = asid; } @@ -120,16 +129,16 @@ * We had a valid ASID in a previous life, so try to re-use * it if possible. */ - asid &= ~ASID_MASK; - if (!__test_and_set_bit(asid, asid_map)) + if (!__test_and_set_bit(asid2idx(asid), asid_map)) return newasid; } /* * Allocate a free ASID. If we can't find one, take a note of the - * currently active ASIDs and mark the TLBs as requiring flushes. - * We always count from ASID #1, as we use ASID #0 when setting a - * reserved TTBR0 for the init_mm. + * currently active ASIDs and mark the TLBs as requiring flushes. We + * always count from ASID #2 (index 1), as we use ASID #0 when setting + * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd + * pairs. */ asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx); if (asid != NUM_USER_ASIDS) @@ -146,7 +155,7 @@ set_asid: __set_bit(asid, asid_map); cur_idx = asid; - return asid | generation; + return idx2asid(asid) | generation; } void check_and_switch_context(struct mm_struct *mm, unsigned int cpu) @@ -182,12 +191,29 @@ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); switch_mm_fastpath: + + arm64_apply_bp_hardening(); + + /* + * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when + * emulating PAN. + */ + if (!system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); } +/* Errata workaround post TTBRx_EL1 update. */ +asmlinkage void post_ttbr_update_workaround(void) +{ + asm(ALTERNATIVE("nop; nop; nop", + "ic iallu; dsb nsh; isb", + ARM64_WORKAROUND_CAVIUM_27456, + CONFIG_CAVIUM_ERRATUM_27456)); +} + static int asids_init(void) { - int fld = cpuid_feature_extract_field(read_cpuid(ID_AA64MMFR0_EL1), 4); + int fld = cpuid_feature_extract_field(read_cpuid(SYS_ID_AA64MMFR0_EL1), 4); switch (fld) { default: diff -ruw linux-4.4.115/arch/arm64/mm/copypage.c linux-4.4.115-fbx/arch/arm64/mm/copypage.c --- linux-4.4.115/arch/arm64/mm/copypage.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/copypage.c 2019-01-22 16:16:21.559228802 +0100 @@ -24,8 +24,9 @@ void __cpu_copy_user_page(void *kto, const void *kfrom, unsigned long vaddr) { + struct page *page = virt_to_page(kto); copy_page(kto, kfrom); - __flush_dcache_area(kto, PAGE_SIZE); + flush_dcache_page(page); } EXPORT_SYMBOL_GPL(__cpu_copy_user_page); diff -ruw linux-4.4.115/arch/arm64/mm/dma-mapping.c linux-4.4.115-fbx/arch/arm64/mm/dma-mapping.c --- linux-4.4.115/arch/arm64/mm/dma-mapping.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/dma-mapping.c 2019-01-22 16:16:21.563228839 +0100 @@ -3,6 +3,7 @@ * * Copyright (C) 2012 ARM Ltd. * Author: Catalin Marinas + * Copyright (c) 2017, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -24,23 +25,51 @@ #include #include #include +#include +#include #include #include #include +#include +#include +#include +#include +#include + +#include "mm.h" + static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot, bool coherent) { - if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) + if (dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs)) + return pgprot_noncached(prot); + else if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) return pgprot_writecombine(prot); return prot; } -static struct gen_pool *atomic_pool; +static bool is_dma_coherent(struct device *dev, struct dma_attrs *attrs) +{ + bool is_coherent; + + if (dma_get_attr(DMA_ATTR_FORCE_COHERENT, attrs)) + is_coherent = true; + else if (dma_get_attr(DMA_ATTR_FORCE_NON_COHERENT, attrs)) + is_coherent = false; + else if (is_device_dma_coherent(dev)) + is_coherent = true; + else + is_coherent = false; + + return is_coherent; +} +static struct gen_pool *atomic_pool; +#define NO_KERNEL_MAPPING_DUMMY 0x2222 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K -static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE; +static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; static int __init early_coherent_pool(char *p) { @@ -86,10 +115,47 @@ return 1; } +static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, + void *data) +{ + struct page *page = virt_to_page(addr); + pgprot_t prot = *(pgprot_t *)data; + + set_pte(pte, mk_pte(page, prot)); + return 0; +} + +static int __dma_clear_pte(pte_t *pte, pgtable_t token, unsigned long addr, + void *data) +{ + pte_clear(&init_mm, addr, pte); + return 0; +} + +static void __dma_remap(struct page *page, size_t size, pgprot_t prot, + bool no_kernel_map) +{ + unsigned long start = (unsigned long) page_address(page); + unsigned end = start + size; + int (*func)(pte_t *pte, pgtable_t token, unsigned long addr, + void *data); + + if (no_kernel_map) + func = __dma_clear_pte; + else + func = __dma_update_pte; + + apply_to_page_range(&init_mm, start, size, func, &prot); + mb(); + flush_tlb_kernel_range(start, end); +} + static void *__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { + void *addr; + if (dev == NULL) { WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); return NULL; @@ -100,7 +166,6 @@ flags |= GFP_DMA; if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) { struct page *page; - void *addr; page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, get_order(size)); @@ -110,10 +175,20 @@ *dma_handle = phys_to_dma(dev, page_to_phys(page)); addr = page_address(page); memset(addr, 0, size); - return addr; } else { - return swiotlb_alloc_coherent(dev, size, dma_handle, flags); + addr = swiotlb_alloc_coherent(dev, size, dma_handle, flags); } + + if (addr && (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs) || + dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs))) { + /* + * flush the caches here because we can't later + */ + __dma_flush_range(addr, addr + size); + __dma_remap(virt_to_page(addr), size, 0, true); + } + + return addr; } static void __dma_free_coherent(struct device *dev, size_t size, @@ -123,11 +198,16 @@ bool freed; phys_addr_t paddr = dma_to_phys(dev, dma_handle); + size = PAGE_ALIGN(size); if (dev == NULL) { WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); return; } + if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs) || + dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs)) + __dma_remap(phys_to_page(paddr), size, PAGE_KERNEL, false); + freed = dma_release_from_contiguous(dev, phys_to_page(paddr), size >> PAGE_SHIFT); @@ -141,8 +221,7 @@ { struct page *page; void *ptr, *coherent_ptr; - bool coherent = is_device_dma_coherent(dev); - pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false); + bool coherent = is_dma_coherent(dev, attrs); size = PAGE_ALIGN(size); @@ -164,16 +243,22 @@ if (coherent) return ptr; + if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) { + coherent_ptr = (void *)NO_KERNEL_MAPPING_DUMMY; + } else { + if (!dma_get_attr(DMA_ATTR_STRONGLY_ORDERED, attrs)) /* remove any dirty cache lines on the kernel alias */ __dma_flush_range(ptr, ptr + size); /* create a coherent mapping */ page = virt_to_page(ptr); coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP, - prot, NULL); + __get_dma_pgprot(attrs, + __pgprot(PROT_NORMAL_NC), false), + NULL); if (!coherent_ptr) goto no_map; - + } return coherent_ptr; no_map: @@ -191,9 +276,10 @@ size = PAGE_ALIGN(size); - if (!is_device_dma_coherent(dev)) { + if (!is_dma_coherent(dev, attrs)) { if (__free_from_pool(vaddr, size)) return; + if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) vunmap(vaddr); } __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs); @@ -207,7 +293,7 @@ dma_addr_t dev_addr; dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs); - if (!is_device_dma_coherent(dev)) + if (!is_dma_coherent(dev, attrs)) __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); return dev_addr; @@ -218,7 +304,7 @@ size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - if (!is_device_dma_coherent(dev)) + if (!is_dma_coherent(dev, attrs)) __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); swiotlb_unmap_page(dev, dev_addr, size, dir, attrs); } @@ -231,7 +317,7 @@ int i, ret; ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs); - if (!is_device_dma_coherent(dev)) + if (!is_dma_coherent(dev, attrs)) for_each_sg(sgl, sg, ret, i) __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), sg->length, dir); @@ -247,7 +333,7 @@ struct scatterlist *sg; int i; - if (!is_device_dma_coherent(dev)) + if (!is_dma_coherent(dev, attrs)) for_each_sg(sgl, sg, nelems, i) __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), sg->length, dir); @@ -313,7 +399,7 @@ unsigned long off = vma->vm_pgoff; vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, - is_device_dma_coherent(dev)); + is_dma_coherent(dev, attrs)); if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) return ret; @@ -341,6 +427,55 @@ return ret; } +static void *arm64_dma_remap(struct device *dev, void *cpu_addr, + dma_addr_t handle, size_t size, + struct dma_attrs *attrs) +{ + struct page *page = phys_to_page(dma_to_phys(dev, handle)); + pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false); + unsigned long offset = handle & ~PAGE_MASK; + struct vm_struct *area; + unsigned long addr; + + size = PAGE_ALIGN(size + offset); + + /* + * DMA allocation can be mapped to user space, so lets + * set VM_USERMAP flags too. + */ + area = get_vm_area(size, VM_USERMAP); + if (!area) + return NULL; + + addr = (unsigned long)area->addr; + area->phys_addr = __pfn_to_phys(page_to_pfn(page)); + + if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) { + vunmap((void *)addr); + return NULL; + } + return (void *)addr + offset; +} + +static void arm64_dma_unremap(struct device *dev, void *remapped_addr, + size_t size) +{ + struct vm_struct *area; + + size = PAGE_ALIGN(size); + remapped_addr = (void *)((unsigned long)remapped_addr & PAGE_MASK); + + area = find_vm_area(remapped_addr); + if (!area) { + WARN(1, "trying to free invalid coherent area: %p\n", + remapped_addr); + return; + } + vunmap(remapped_addr); + flush_tlb_kernel_range((unsigned long)remapped_addr, + (unsigned long)(remapped_addr + size)); +} + static struct dma_map_ops swiotlb_dma_ops = { .alloc = __dma_alloc, .free = __dma_free, @@ -356,6 +491,8 @@ .sync_sg_for_device = __swiotlb_sync_sg_for_device, .dma_supported = swiotlb_dma_supported, .mapping_error = swiotlb_dma_mapping_error, + .remap = arm64_dma_remap, + .unremap = arm64_dma_unremap, }; static int __init atomic_pool_init(void) @@ -406,7 +543,7 @@ goto out; remove_mapping: - dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP); + dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP, true); destroy_genpool: gen_pool_destroy(atomic_pool); atomic_pool = NULL; @@ -427,6 +564,7 @@ dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { + WARN(1, "dma alloc failure, device may be missing a call to arch_setup_dma_ops"); return NULL; } @@ -542,7 +680,7 @@ dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) { - bool coherent = is_device_dma_coherent(dev); + bool coherent = is_dma_coherent(dev, attrs); int ioprot = dma_direction_to_prot(DMA_BIDIRECTIONAL, coherent); size_t iosize = size; void *addr; @@ -624,7 +762,7 @@ if (WARN_ON(!area || !area->pages)) return; iommu_dma_free(dev, area->pages, iosize, &handle); - dma_common_free_remap(cpu_addr, size, VM_USERMAP); + dma_common_free_remap(cpu_addr, size, VM_USERMAP, true); } else { iommu_dma_unmap_page(dev, handle, iosize, 0, NULL); __free_pages(virt_to_page(cpu_addr), get_order(size)); @@ -639,7 +777,7 @@ int ret; vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, - is_device_dma_coherent(dev)); + is_dma_coherent(dev, attrs)); if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) return ret; @@ -696,7 +834,7 @@ enum dma_data_direction dir, struct dma_attrs *attrs) { - bool coherent = is_device_dma_coherent(dev); + bool coherent = is_dma_coherent(dev, attrs); int prot = dma_direction_to_prot(dir, coherent); dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot); @@ -749,7 +887,7 @@ int nelems, enum dma_data_direction dir, struct dma_attrs *attrs) { - bool coherent = is_device_dma_coherent(dev); + bool coherent = is_dma_coherent(dev, attrs); if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) __iommu_sync_sg_for_device(dev, sgl, nelems, dir); @@ -782,7 +920,6 @@ .sync_single_for_device = __iommu_sync_single_for_device, .sync_sg_for_cpu = __iommu_sync_sg_for_cpu, .sync_sg_for_device = __iommu_sync_sg_for_device, - .dma_supported = iommu_dma_supported, .mapping_error = iommu_dma_mapping_error, }; @@ -896,7 +1033,7 @@ return 0; } -static int register_iommu_dma_ops_notifier(struct bus_type *bus) +static int __init register_iommu_dma_ops_notifier(struct bus_type *bus) { struct notifier_block *nb = kzalloc(sizeof(*nb), GFP_KERNEL); int ret; @@ -993,3 +1130,922 @@ dev->archdata.dma_coherent = coherent; __iommu_setup_dma_ops(dev, dma_base, size, iommu); } +EXPORT_SYMBOL(arch_setup_dma_ops); + +#ifdef CONFIG_ARM64_DMA_USE_IOMMU + +static int __get_iommu_pgprot(struct dma_attrs *attrs, int prot, + bool coherent) +{ + if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs)) + prot |= IOMMU_NOEXEC; + if (coherent) + prot |= IOMMU_CACHE; + + return prot; +} + +/* + * Make an area consistent for devices. + * Note: Drivers should NOT use this function directly, as it will break + * platforms with CONFIG_DMABOUNCE. + * Use the driver DMA support - see dma-mapping.h (dma_sync_*) + */ +static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, + size_t size, enum dma_data_direction dir) +{ + __dma_map_area(page_address(page) + off, size, dir); +} + +static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, + size_t size, enum dma_data_direction dir) +{ + __dma_unmap_area(page_address(page) + off, size, dir); + + /* + * Mark the D-cache clean for this page to avoid extra flushing. + */ + if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) + set_bit(PG_dcache_clean, &page->flags); +} + +static int arm_dma_set_mask(struct device *dev, u64 dma_mask) +{ + if (!dev->dma_mask || !dma_supported(dev, dma_mask)) + return -EIO; + + *dev->dma_mask = dma_mask; + + return 0; +} + +/* IOMMU */ + +static void __dma_clear_buffer(struct page *page, size_t size, + struct dma_attrs *attrs, bool is_coherent) +{ + /* + * Ensure that the allocated pages are zeroed, and that any data + * lurking in the kernel direct-mapped region is invalidated. + */ + void *ptr = page_address(page); + if (!dma_get_attr(DMA_ATTR_SKIP_ZEROING, attrs)) + memset(ptr, 0, size); + if (!is_coherent) + dmac_flush_range(ptr, ptr + size); +} + +static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, + size_t size) +{ + unsigned int order = get_order(size); + unsigned int align = 0; + unsigned int count, start; + unsigned long flags; + + if (order > CONFIG_ARM64_DMA_IOMMU_ALIGNMENT) + order = CONFIG_ARM64_DMA_IOMMU_ALIGNMENT; + + count = PAGE_ALIGN(size) >> PAGE_SHIFT; + align = (1 << order) - 1; + + spin_lock_irqsave(&mapping->lock, flags); + start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0, + count, align); + if (start > mapping->bits) { + spin_unlock_irqrestore(&mapping->lock, flags); + return DMA_ERROR_CODE; + } + + bitmap_set(mapping->bitmap, start, count); + spin_unlock_irqrestore(&mapping->lock, flags); + + return mapping->base + (start << PAGE_SHIFT); +} + +static inline void __free_iova(struct dma_iommu_mapping *mapping, + dma_addr_t addr, size_t size) +{ + unsigned int start = (addr - mapping->base) >> PAGE_SHIFT; + unsigned int count = size >> PAGE_SHIFT; + unsigned long flags; + + spin_lock_irqsave(&mapping->lock, flags); + bitmap_clear(mapping->bitmap, start, count); + spin_unlock_irqrestore(&mapping->lock, flags); +} + +static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, + gfp_t gfp, struct dma_attrs *attrs) +{ + struct page **pages; + size_t count = size >> PAGE_SHIFT; + size_t array_size = count * sizeof(struct page *); + int i = 0; + bool is_coherent = is_dma_coherent(dev, attrs); + + if (array_size <= PAGE_SIZE) + pages = kzalloc(array_size, gfp); + else + pages = vzalloc(array_size); + if (!pages) + return NULL; + + if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { + unsigned long order = get_order(size); + struct page *page; + + page = dma_alloc_from_contiguous(dev, count, order); + if (!page) + goto error; + + __dma_clear_buffer(page, size, attrs, is_coherent); + + for (i = 0; i < count; i++) + pages[i] = page + i; + + return pages; + } + + /* + * IOMMU can map any pages, so himem can also be used here + */ + gfp |= __GFP_NOWARN | __GFP_HIGHMEM; + + while (count) { + int j, order = __fls(count); + + pages[i] = alloc_pages(gfp, order); + while (!pages[i] && order) + pages[i] = alloc_pages(gfp, --order); + if (!pages[i]) + goto error; + + if (order) { + split_page(pages[i], order); + j = 1 << order; + while (--j) + pages[i + j] = pages[i] + j; + } + + __dma_clear_buffer(pages[i], PAGE_SIZE << order, attrs, + is_coherent); + i += 1 << order; + count -= 1 << order; + } + + return pages; +error: + while (i--) + if (pages[i]) + __free_pages(pages[i], 0); + if (array_size <= PAGE_SIZE) + kfree(pages); + else + vfree(pages); + return NULL; +} + +static int __iommu_free_buffer(struct device *dev, struct page **pages, + size_t size, struct dma_attrs *attrs) +{ + int count = size >> PAGE_SHIFT; + int array_size = count * sizeof(struct page *); + int i; + + if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) { + dma_release_from_contiguous(dev, pages[0], count); + } else { + for (i = 0; i < count; i++) + if (pages[i]) + __free_pages(pages[i], 0); + } + + if (array_size <= PAGE_SIZE) + kfree(pages); + else + vfree(pages); + return 0; +} + +/* + * Create a CPU mapping for a specified pages + */ +static void * +__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, + const void *caller) +{ + return dma_common_pages_remap(pages, size, VM_USERMAP, prot, caller); +} + +/* + * Create a mapping in device IO address space for specified pages + */ +static dma_addr_t __iommu_create_mapping(struct device *dev, + struct page **pages, size_t size, + struct dma_attrs *attrs) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + dma_addr_t dma_addr, iova; + int i, ret; + int prot = IOMMU_READ | IOMMU_WRITE; + + dma_addr = __alloc_iova(mapping, size); + if (dma_addr == DMA_ERROR_CODE) + return dma_addr; + prot = __get_iommu_pgprot(attrs, prot, + is_dma_coherent(dev, attrs)); + + iova = dma_addr; + for (i = 0; i < count; ) { + unsigned int next_pfn = page_to_pfn(pages[i]) + 1; + phys_addr_t phys = page_to_phys(pages[i]); + unsigned int len, j; + + for (j = i + 1; j < count; j++, next_pfn++) + if (page_to_pfn(pages[j]) != next_pfn) + break; + + len = (j - i) << PAGE_SHIFT; + ret = iommu_map(mapping->domain, iova, phys, len, prot); + if (ret < 0) + goto fail; + iova += len; + i = j; + } + return dma_addr; +fail: + iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); + __free_iova(mapping, dma_addr, size); + return DMA_ERROR_CODE; +} + +static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, + size_t size) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + + /* + * add optional in-page offset from iova to size and align + * result to page size + */ + size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); + iova &= PAGE_MASK; + + iommu_unmap(mapping->domain, iova, size); + __free_iova(mapping, iova, size); + return 0; +} + +static struct page **__atomic_get_pages(void *addr) +{ + struct page *page; + phys_addr_t phys; + + phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr); + page = phys_to_page(phys); + + return (struct page **)page; +} + +static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs) +{ + struct vm_struct *area; + + if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) + return __atomic_get_pages(cpu_addr); + + if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) + return cpu_addr; + + area = find_vm_area(cpu_addr); + if (area) + return area->pages; + return NULL; +} + +static void *__iommu_alloc_atomic(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, + struct dma_attrs *attrs) +{ + struct page *page; + struct page **pages; + size_t count = size >> PAGE_SHIFT; + size_t array_size = count * sizeof(struct page *); + int i; + void *addr; + bool coherent = is_dma_coherent(dev, attrs); + + if (array_size <= PAGE_SIZE) + pages = kzalloc(array_size, gfp); + else + pages = vzalloc(array_size); + + if (!pages) + return NULL; + + if (coherent) { + page = alloc_pages(gfp, get_order(size)); + addr = page ? page_address(page) : NULL; + } else { + addr = __alloc_from_pool(size, &page, gfp); + } + + if (!addr) + goto err_free; + + for (i = 0; i < count ; i++) + pages[i] = page + i; + + *handle = __iommu_create_mapping(dev, pages, size, attrs); + if (*handle == DMA_ERROR_CODE) + goto err_mapping; + + kvfree(pages); + return addr; + +err_mapping: + if (coherent) + __free_pages(page, get_order(size)); + else + __free_from_pool(addr, size); +err_free: + kvfree(pages); + return NULL; +} + +static void __iommu_free_atomic(struct device *dev, void *cpu_addr, + dma_addr_t handle, size_t size) +{ + __iommu_remove_mapping(dev, handle, size); + __free_from_pool(cpu_addr, size); +} + +static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) +{ + bool coherent = is_dma_coherent(dev, attrs); + pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent); + struct page **pages; + void *addr = NULL; + + *handle = DMA_ERROR_CODE; + size = PAGE_ALIGN(size); + + if (!gfpflags_allow_blocking(gfp)) + return __iommu_alloc_atomic(dev, size, handle, gfp, attrs); + + /* + * Following is a work-around (a.k.a. hack) to prevent pages + * with __GFP_COMP being passed to split_page() which cannot + * handle them. The real problem is that this flag probably + * should be 0 on ARM as it is not supported on this + * platform; see CONFIG_HUGETLBFS. + */ + gfp &= ~(__GFP_COMP); + + pages = __iommu_alloc_buffer(dev, size, gfp, attrs); + if (!pages) + return NULL; + + *handle = __iommu_create_mapping(dev, pages, size, attrs); + if (*handle == DMA_ERROR_CODE) + goto err_buffer; + + if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) + return pages; + + addr = __iommu_alloc_remap(pages, size, gfp, prot, + __builtin_return_address(0)); + if (!addr) + goto err_mapping; + + return addr; + +err_mapping: + __iommu_remove_mapping(dev, *handle, size); +err_buffer: + __iommu_free_buffer(dev, pages, size, attrs); + return NULL; +} + +static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_attrs *attrs) +{ + unsigned long uaddr = vma->vm_start; + unsigned long usize = vma->vm_end - vma->vm_start; + struct page **pages = __iommu_get_pages(cpu_addr, attrs); + bool coherent = is_dma_coherent(dev, attrs); + + vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, + coherent); + + if (!pages) + return -ENXIO; + + do { + int ret = vm_insert_page(vma, uaddr, *pages++); + if (ret) { + pr_err("Remapping memory failed: %d\n", ret); + return ret; + } + uaddr += PAGE_SIZE; + usize -= PAGE_SIZE; + } while (usize > 0); + + return 0; +} + +/* + * free a page as defined by the above mapping. + * Must not be called with IRQs disabled. + */ +void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, struct dma_attrs *attrs) +{ + struct page **pages; + size = PAGE_ALIGN(size); + + if (__in_atomic_pool(cpu_addr, size)) { + __iommu_free_atomic(dev, cpu_addr, handle, size); + return; + } + + pages = __iommu_get_pages(cpu_addr, attrs); + if (!pages) { + WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); + return; + } + + if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) + dma_common_free_remap(cpu_addr, size, VM_USERMAP, true); + + __iommu_remove_mapping(dev, handle, size); + __iommu_free_buffer(dev, pages, size, attrs); +} + +int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, + size_t size, struct dma_attrs *attrs) +{ + unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + struct page **pages = __iommu_get_pages(cpu_addr, attrs); + + if (!pages) + return -ENXIO; + + return sg_alloc_table_from_pages(sgt, pages, count, 0, size, + GFP_KERNEL); +} + +static int __dma_direction_to_prot(enum dma_data_direction dir) +{ + int prot; + + switch (dir) { + case DMA_BIDIRECTIONAL: + prot = IOMMU_READ | IOMMU_WRITE; + break; + case DMA_TO_DEVICE: + prot = IOMMU_READ; + break; + case DMA_FROM_DEVICE: + prot = IOMMU_WRITE; + break; + default: + prot = 0; + } + + return prot; +} + +/** + * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA + * @dev: valid struct device pointer + * @sg: list of buffers + * @nents: number of buffers to map + * @dir: DMA transfer direction + * + * Map a set of buffers described by scatterlist in streaming mode for DMA. + * The scatter gather list elements are merged together (if possible) and + * tagged with the appropriate dma address and length. They are obtained via + * sg_dma_{address,length}. + */ +int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, struct dma_attrs *attrs) +{ + struct scatterlist *s; + int ret, i; + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + unsigned int total_length = 0, current_offset = 0; + dma_addr_t iova; + int prot = __dma_direction_to_prot(dir); + + for_each_sg(sg, s, nents, i) + total_length += s->length; + + iova = __alloc_iova(mapping, total_length); + if (iova == DMA_ERROR_CODE) { + dev_err(dev, "Couldn't allocate iova for sg %p\n", sg); + return 0; + } + prot = __get_iommu_pgprot(attrs, prot, + is_dma_coherent(dev, attrs)); + + ret = iommu_map_sg(mapping->domain, iova, sg, nents, prot); + if (ret != total_length) { + __free_iova(mapping, iova, total_length); + return 0; + } + + for_each_sg(sg, s, nents, i) { + s->dma_address = iova + current_offset; + s->dma_length = total_length - current_offset; + current_offset += s->length; + } + + return nents; +} + +/** + * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg + * @dev: valid struct device pointer + * @sg: list of buffers + * @nents: number of buffers to unmap (same as was passed to dma_map_sg) + * @dir: DMA transfer direction (same as was passed to dma_map_sg) + * + * Unmap a set of streaming mode DMA translations. Again, CPU access + * rules concerning calls here are the same as for dma_unmap_single(). + */ +void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + unsigned int total_length = sg_dma_len(sg); + dma_addr_t iova = sg_dma_address(sg); + + total_length = PAGE_ALIGN((iova & ~PAGE_MASK) + total_length); + iova &= PAGE_MASK; + + iommu_unmap(mapping->domain, iova, total_length); + __free_iova(mapping, iova, total_length); +} + +/** + * arm_iommu_sync_sg_for_cpu + * @dev: valid struct device pointer + * @sg: list of buffers + * @nents: number of buffers to map (returned from dma_map_sg) + * @dir: DMA transfer direction (same as was passed to dma_map_sg) + */ +void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir) +{ + struct scatterlist *s; + int i; + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + dma_addr_t iova = sg_dma_address(sg); + bool iova_coherent = iommu_is_iova_coherent(mapping->domain, iova); + + if (iova_coherent) + return; + + for_each_sg(sg, s, nents, i) + __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); + +} + +/** + * arm_iommu_sync_sg_for_device + * @dev: valid struct device pointer + * @sg: list of buffers + * @nents: number of buffers to map (returned from dma_map_sg) + * @dir: DMA transfer direction (same as was passed to dma_map_sg) + */ +void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir) +{ + struct scatterlist *s; + int i; + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + dma_addr_t iova = sg_dma_address(sg); + bool iova_coherent = iommu_is_iova_coherent(mapping->domain, iova); + + if (iova_coherent) + return; + + for_each_sg(sg, s, nents, i) + __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); +} + + +/** + * arm_coherent_iommu_map_page + * @dev: valid struct device pointer + * @page: page that buffer resides in + * @offset: offset into page for start of buffer + * @size: size of buffer to map + * @dir: DMA transfer direction + * + * Coherent IOMMU aware version of arm_dma_map_page() + */ +static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, + struct page *page, unsigned long offset, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + dma_addr_t dma_addr; + int ret, prot, len, start_offset, map_offset; + + map_offset = offset & ~PAGE_MASK; + start_offset = offset & PAGE_MASK; + len = PAGE_ALIGN(map_offset + size); + + dma_addr = __alloc_iova(mapping, len); + if (dma_addr == DMA_ERROR_CODE) + return dma_addr; + + prot = __dma_direction_to_prot(dir); + prot = __get_iommu_pgprot(attrs, prot, + is_dma_coherent(dev, attrs)); + + ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page) + + start_offset, len, prot); + if (ret < 0) + goto fail; + + return dma_addr + map_offset; +fail: + __free_iova(mapping, dma_addr, len); + return DMA_ERROR_CODE; +} + +/** + * arm_iommu_map_page + * @dev: valid struct device pointer + * @page: page that buffer resides in + * @offset: offset into page for start of buffer + * @size: size of buffer to map + * @dir: DMA transfer direction + * + * IOMMU aware version of arm_dma_map_page() + */ +static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs) +{ + if (!is_dma_coherent(dev, attrs) && + !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)) + __dma_page_cpu_to_dev(page, offset, size, dir); + + return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); +} + +/** + * arm_iommu_unmap_page + * @dev: valid struct device pointer + * @handle: DMA address of buffer + * @size: size of buffer (same as passed to dma_map_page) + * @dir: DMA transfer direction (same as passed to dma_map_page) + * + * IOMMU aware version of arm_dma_unmap_page() + */ +static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + dma_addr_t iova = handle & PAGE_MASK; + struct page *page = phys_to_page(iommu_iova_to_phys( + mapping->domain, iova)); + int offset = handle & ~PAGE_MASK; + int len = PAGE_ALIGN(size + offset); + + if (!(is_dma_coherent(dev, attrs) || + dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))) + __dma_page_dev_to_cpu(page, offset, size, dir); + + iommu_unmap(mapping->domain, iova, len); + __free_iova(mapping, iova, len); +} + +static void arm_iommu_sync_single_for_cpu(struct device *dev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + dma_addr_t iova = handle & PAGE_MASK; + struct page *page = phys_to_page(iommu_iova_to_phys( + mapping->domain, iova)); + unsigned int offset = handle & ~PAGE_MASK; + bool iova_coherent = iommu_is_iova_coherent(mapping->domain, handle); + + if (!iova_coherent) + __dma_page_dev_to_cpu(page, offset, size, dir); +} + +static void arm_iommu_sync_single_for_device(struct device *dev, + dma_addr_t handle, size_t size, enum dma_data_direction dir) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + dma_addr_t iova = handle & PAGE_MASK; + struct page *page = phys_to_page(iommu_iova_to_phys( + mapping->domain, iova)); + unsigned int offset = handle & ~PAGE_MASK; + bool iova_coherent = iommu_is_iova_coherent(mapping->domain, handle); + + if (!iova_coherent) + __dma_page_cpu_to_dev(page, offset, size, dir); +} + +static int arm_iommu_dma_supported(struct device *dev, u64 mask) +{ + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); + + if (!mapping) { + dev_warn(dev, "No IOMMU mapping for device\n"); + return 0; + } + + return iommu_dma_supported(mapping->domain, dev, mask); +} + +static int arm_iommu_mapping_error(struct device *dev, + dma_addr_t dma_addr) +{ + return dma_addr == DMA_ERROR_CODE; +} + +const struct dma_map_ops iommu_ops = { + .alloc = arm_iommu_alloc_attrs, + .free = arm_iommu_free_attrs, + .mmap = arm_iommu_mmap_attrs, + .get_sgtable = arm_iommu_get_sgtable, + + .map_page = arm_iommu_map_page, + .unmap_page = arm_iommu_unmap_page, + .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, + .sync_single_for_device = arm_iommu_sync_single_for_device, + + .map_sg = arm_iommu_map_sg, + .unmap_sg = arm_iommu_unmap_sg, + .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, + .sync_sg_for_device = arm_iommu_sync_sg_for_device, + + .set_dma_mask = arm_dma_set_mask, + .dma_supported = arm_iommu_dma_supported, + .mapping_error = arm_iommu_mapping_error, +}; + +/** + * arm_iommu_create_mapping + * @bus: pointer to the bus holding the client device (for IOMMU calls) + * @base: start address of the valid IO address space + * @size: maximum size of the valid IO address space + * + * Creates a mapping structure which holds information about used/unused + * IO address ranges, which is required to perform memory allocation and + * mapping with IOMMU aware functions. + * + * The client device need to be attached to the mapping with + * arm_iommu_attach_device function. + */ +struct dma_iommu_mapping * +arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size) +{ + unsigned int bits = size >> PAGE_SHIFT; + unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); + struct dma_iommu_mapping *mapping; + int err = -ENOMEM; + + if (!bitmap_size) + return ERR_PTR(-EINVAL); + + mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); + if (!mapping) + goto err; + + mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL | __GFP_NOWARN | + __GFP_NORETRY); + if (!mapping->bitmap) + mapping->bitmap = vzalloc(bitmap_size); + + if (!mapping->bitmap) + goto err2; + + mapping->base = base; + mapping->bits = bits; + spin_lock_init(&mapping->lock); + + mapping->domain = iommu_domain_alloc(bus); + if (!mapping->domain) + goto err3; + + kref_init(&mapping->kref); + return mapping; +err3: + kvfree(mapping->bitmap); +err2: + kfree(mapping); +err: + return ERR_PTR(err); +} +EXPORT_SYMBOL(arm_iommu_create_mapping); + +static void release_iommu_mapping(struct kref *kref) +{ + struct dma_iommu_mapping *mapping = + container_of(kref, struct dma_iommu_mapping, kref); + + iommu_domain_free(mapping->domain); + kvfree(mapping->bitmap); + kfree(mapping); +} + +void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) +{ + if (mapping) + kref_put(&mapping->kref, release_iommu_mapping); +} +EXPORT_SYMBOL(arm_iommu_release_mapping); + +/** + * arm_iommu_attach_device + * @dev: valid struct device pointer + * @mapping: io address space mapping structure (returned from + * arm_iommu_create_mapping) + * + * Attaches specified io address space mapping to the provided device, + * this replaces the dma operations (dma_map_ops pointer) with the + * IOMMU aware version. More than one client might be attached to + * the same io address space mapping. + */ +int arm_iommu_attach_device(struct device *dev, + struct dma_iommu_mapping *mapping) +{ + int err; + int s1_bypass = 0, is_fast = 0; + + iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_FAST, &is_fast); + if (is_fast) + return fast_smmu_attach_device(dev, mapping); + + err = iommu_attach_device(mapping->domain, dev); + if (err) + return err; + + iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_S1_BYPASS, + &s1_bypass); + + kref_get(&mapping->kref); + dev->archdata.mapping = mapping; + if (!s1_bypass) + set_dma_ops(dev, &iommu_ops); + + pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); + return 0; +} +EXPORT_SYMBOL(arm_iommu_attach_device); + +/** + * arm_iommu_detach_device + * @dev: valid struct device pointer + * + * Detaches the provided device from a previously attached map. + * This voids the dma operations (dma_map_ops pointer) + */ +void arm_iommu_detach_device(struct device *dev) +{ + struct dma_iommu_mapping *mapping; + int is_fast, s1_bypass = 0; + + mapping = to_dma_iommu_mapping(dev); + if (!mapping) { + dev_warn(dev, "Not attached\n"); + return; + } + + iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_FAST, &is_fast); + if (is_fast) { + fast_smmu_detach_device(dev, mapping); + return; + } + + iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_S1_BYPASS, + &s1_bypass); + + if (msm_dma_unmap_all_for_dev(dev)) + dev_warn(dev, "IOMMU detach with outstanding mappings\n"); + + iommu_detach_device(mapping->domain, dev); + kref_put(&mapping->kref, release_iommu_mapping); + dev->archdata.mapping = NULL; + if (!s1_bypass) + set_dma_ops(dev, NULL); + + pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); +} +EXPORT_SYMBOL(arm_iommu_detach_device); + +#endif diff -ruw linux-4.4.115/arch/arm64/mm/extable.c linux-4.4.115-fbx/arch/arm64/mm/extable.c --- linux-4.4.115/arch/arm64/mm/extable.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/extable.c 2019-01-22 16:16:21.563228839 +0100 @@ -11,7 +11,7 @@ fixup = search_exception_tables(instruction_pointer(regs)); if (fixup) - regs->pc = fixup->fixup; + regs->pc = (unsigned long)&fixup->fixup + fixup->fixup; return fixup != NULL; } diff -ruw linux-4.4.115/arch/arm64/mm/fault.c linux-4.4.115-fbx/arch/arm64/mm/fault.c --- linux-4.4.115/arch/arm64/mm/fault.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/fault.c 2019-10-29 09:26:23.009196974 +0100 @@ -40,9 +40,35 @@ #include #include #include +#include +#include + +#include static const char *fault_name(unsigned int esr); +#ifdef CONFIG_KPROBES +static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) +{ + int ret = 0; + + /* kprobe_running() needs smp_processor_id() */ + if (!user_mode(regs)) { + preempt_disable(); + if (kprobe_running() && kprobe_fault_handler(regs, esr)) + ret = 1; + preempt_enable(); + } + + return ret; +} +#else +static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) +{ + return 0; +} +#endif + /* * Dump out the page tables associated with 'addr' in mm 'mm'. */ @@ -133,6 +159,11 @@ } #endif +static bool is_el1_instruction_abort(unsigned int esr) +{ + return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; +} + /* * The kernel tried to access some page that wasn't present. */ @@ -141,8 +172,9 @@ { /* * Are we prepared to handle this kernel fault? + * We are almost certainly not prepared to handle instruction faults. */ - if (fixup_exception(regs)) + if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) return; /* @@ -169,6 +201,8 @@ { struct siginfo si; + trace_user_fault(tsk, addr, esr); + if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n", tsk->comm, task_pid_nr(tsk), fault_name(esr), sig, @@ -204,8 +238,6 @@ #define VM_FAULT_BADMAP 0x010000 #define VM_FAULT_BADACCESS 0x020000 -#define ESR_LNX_EXEC (1 << 24) - static int __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int mm_flags, unsigned long vm_flags, struct task_struct *tsk) @@ -244,6 +276,26 @@ return fault; } +static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs) +{ + unsigned int ec = ESR_ELx_EC(esr); + unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; + + if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) + return false; + + if (system_uses_ttbr0_pan()) + return fsc_type == ESR_ELx_FSC_FAULT && + (regs->pstate & PSR_PAN_BIT); + else + return fsc_type == ESR_ELx_FSC_PERM; +} + +static bool is_el0_instruction_abort(unsigned int esr) +{ + return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; +} + static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { @@ -253,6 +305,9 @@ unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC; unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + if (notify_page_fault(regs, esr)) + return 0; + tsk = current; mm = tsk->mm; @@ -270,19 +325,21 @@ if (user_mode(regs)) mm_flags |= FAULT_FLAG_USER; - if (esr & ESR_LNX_EXEC) { + if (is_el0_instruction_abort(esr)) { vm_flags = VM_EXEC; - } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { + } else if (((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) || + ((esr & ESR_ELx_CM) && !(mm_flags & FAULT_FLAG_USER))) { vm_flags = VM_WRITE; mm_flags |= FAULT_FLAG_WRITE; } - /* - * PAN bit set implies the fault happened in kernel space, but not - * in the arch's user access functions. - */ - if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT)) - goto no_context; + if (addr < USER_DS && is_permission_fault(esr, regs)) { + if (is_el1_instruction_abort(esr)) + die("Attempting to execute userspace memory", regs, esr); + + if (!search_exception_tables(regs->pc)) + die("Accessing user space memory outside uaccess.h routines", regs, esr); + } /* * As per x86, we may deadlock here. However, since the kernel only @@ -399,6 +456,19 @@ } /* + * TLB conflict is already handled in EL2. This rourtine should return zero + * so that, do_mem_abort would not crash kernel thinking TLB conflict not + * handled. +*/ +#ifdef CONFIG_QCOM_TLB_EL2_HANDLER +static int do_tlb_conf_fault(unsigned long addr, + unsigned int esr, + struct pt_regs *regs) +{ + return 0; +} +#endif +/* * First Level Translation Fault Handler * * We enter here because the first level page table doesn't contain a valid @@ -431,10 +501,11 @@ */ static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) { + arm64_check_cache_ecc(NULL); return 1; } -static struct fault_info { +static const struct fault_info { int (*fn)(unsigned long addr, unsigned int esr, struct pt_regs *regs); int sig; int code; @@ -488,7 +559,11 @@ { do_bad, SIGBUS, 0, "unknown 45" }, { do_bad, SIGBUS, 0, "unknown 46" }, { do_bad, SIGBUS, 0, "unknown 47" }, +#ifdef CONFIG_QCOM_TLB_EL2_HANDLER + { do_tlb_conf_fault, SIGBUS, 0, "TLB conflict abort" }, +#else { do_bad, SIGBUS, 0, "TLB conflict abort" }, +#endif { do_bad, SIGBUS, 0, "unknown 49" }, { do_bad, SIGBUS, 0, "unknown 50" }, { do_bad, SIGBUS, 0, "unknown 51" }, @@ -534,6 +609,22 @@ arm64_notify_die("", regs, &info, esr); } +asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr, + unsigned int esr, + struct pt_regs *regs) +{ + /* + * We've taken an instruction abort from userspace and not yet + * re-enabled IRQs. If the address is a kernel address, apply + * BP hardening prior to enabling IRQs and pre-emption. + */ + if (addr > TASK_SIZE) + arm64_apply_bp_hardening(); + + local_irq_enable(); + do_mem_abort(addr, esr, regs); +} + /* * Handle stack alignment exceptions. */ @@ -609,6 +700,7 @@ return 0; } +NOKPROBE_SYMBOL(do_debug_exception); #ifdef CONFIG_ARM64_PAN int cpu_enable_pan(void *__unused) @@ -624,3 +716,17 @@ return 0; } #endif /* CONFIG_ARM64_PAN */ + +#ifdef CONFIG_ARM64_UAO +/* + * Kernel threads have fs=KERNEL_DS by default, and don't need to call + * set_fs(), devtmpfs in particular relies on this behaviour. + * We need to enable the feature at runtime (instead of adding it to + * PSR_MODE_EL1h) as the feature may not be implemented by the cpu. + */ +int cpu_enable_uao(void *__unused) +{ + asm(SET_PSTATE_UAO(1)); + return 0; +} +#endif /* CONFIG_ARM64_UAO */ diff -ruw linux-4.4.115/arch/arm64/mm/flush.c linux-4.4.115-fbx/arch/arm64/mm/flush.c --- linux-4.4.115/arch/arm64/mm/flush.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/flush.c 2019-10-29 09:26:23.013197013 +0100 @@ -34,19 +34,24 @@ __flush_icache_all(); } -static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, - unsigned long uaddr, void *kaddr, - unsigned long len) +static void sync_icache_aliases(void *kaddr, unsigned long len) { - if (vma->vm_flags & VM_EXEC) { unsigned long addr = (unsigned long)kaddr; + if (icache_is_aliasing()) { - __flush_dcache_area(kaddr, len); + __clean_dcache_area_pou(kaddr, len); __flush_icache_all(); } else { flush_icache_range(addr, addr + len); } } + +static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, + unsigned long uaddr, void *kaddr, + unsigned long len) +{ + if (vma->vm_flags & VM_EXEC) + sync_icache_aliases(kaddr, len); } /* @@ -74,13 +79,11 @@ if (!page_mapping(page)) return; - if (!test_and_set_bit(PG_dcache_clean, &page->flags)) { - __flush_dcache_area(page_address(page), + if (!test_and_set_bit(PG_dcache_clean, &page->flags)) + sync_icache_aliases(page_address(page), PAGE_SIZE << compound_order(page)); + else if (icache_is_aivivt()) __flush_icache_all(); - } else if (icache_is_aivivt()) { - __flush_icache_all(); - } } /* @@ -98,6 +101,7 @@ /* * Additional functions defined in assembly. */ +EXPORT_SYMBOL(flush_cache_all); EXPORT_SYMBOL(flush_icache_range); #ifdef CONFIG_TRANSPARENT_HUGEPAGE diff -ruw linux-4.4.115/arch/arm64/mm/init.c linux-4.4.115-fbx/arch/arm64/mm/init.c --- linux-4.4.115/arch/arm64/mm/init.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/init.c 2019-10-29 09:26:23.013197013 +0100 @@ -34,8 +34,12 @@ #include #include #include +#include +#include #include +#include +#include #include #include #include @@ -45,7 +49,13 @@ #include "mm.h" -phys_addr_t memstart_addr __read_mostly = 0; +/* + * We need to be able to catch inadvertent references to memstart_addr + * that occur (potentially in generic code) before arm64_memblock_init() + * executes, which assigns it its actual value. So use a default value + * that cannot be mistaken for a real physical address. + */ +s64 memstart_addr __read_mostly = -1; phys_addr_t arm64_dma_phys_limit __read_mostly; #ifdef CONFIG_BLK_DEV_INITRD @@ -58,8 +68,8 @@ if (*endp == ',') { size = memparse(endp + 1, NULL); - initrd_start = (unsigned long)__va(start); - initrd_end = (unsigned long)__va(start + size); + initrd_start = start; + initrd_end = start + size; } return 0; } @@ -71,7 +81,7 @@ * currently assumes that for memory starting above 4G, 32-bit devices will * use a DMA offset. */ -static phys_addr_t max_zone_dma_phys(void) +static phys_addr_t __init max_zone_dma_phys(void) { phys_addr_t offset = memblock_start_of_DRAM() & GENMASK_ULL(63, 32); return min(offset + (1ULL << 32), memblock_end_of_DRAM()); @@ -118,19 +128,21 @@ } #ifdef CONFIG_HAVE_ARCH_PFN_VALID +#define PFN_MASK ((1UL << (64 - PAGE_SHIFT)) - 1) + int pfn_valid(unsigned long pfn) { - return memblock_is_memory(pfn << PAGE_SHIFT); + return (pfn & PFN_MASK) == pfn && memblock_is_map_memory(pfn << PAGE_SHIFT); } EXPORT_SYMBOL(pfn_valid); #endif #ifndef CONFIG_SPARSEMEM -static void arm64_memory_present(void) +static void __init arm64_memory_present(void) { } #else -static void arm64_memory_present(void) +static void __init arm64_memory_present(void) { struct memblock_region *reg; @@ -141,6 +153,7 @@ #endif static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX; +static phys_addr_t bootloader_memory_limit; /* * Limit the memory size that was specified via FDT. @@ -159,16 +172,80 @@ void __init arm64_memblock_init(void) { + const s64 linear_region_size = -(s64)PAGE_OFFSET; + + /* + * Ensure that the linear region takes up exactly half of the kernel + * virtual address space. This way, we can distinguish a linear address + * from a kernel/module/vmalloc address by testing a single bit. + */ + BUILD_BUG_ON(linear_region_size != BIT(VA_BITS - 1)); + + /* + * Select a suitable value for the base of physical memory. + */ + memstart_addr = round_down(memblock_start_of_DRAM(), + ARM64_MEMSTART_ALIGN); + + /* + * Remove the memory that we will not be able to cover with the + * linear mapping. Take care not to clip the kernel which may be + * high in memory. + */ + memblock_remove(max_t(u64, memstart_addr + linear_region_size, + __pa_symbol(_end)), ULLONG_MAX); + if (memstart_addr + linear_region_size < memblock_end_of_DRAM()) { + /* ensure that memstart_addr remains sufficiently aligned */ + memstart_addr = round_up(memblock_end_of_DRAM() - linear_region_size, + ARM64_MEMSTART_ALIGN); + memblock_remove(0, memstart_addr); + } + + /* + * Apply the memory limit if it was set. Since the kernel may be loaded + * high up in memory, add back the kernel region that must be accessible + * via the linear mapping. + */ + if (memory_limit != (phys_addr_t)ULLONG_MAX) { + /* + * Save bootloader imposed memory limit before we overwirte + * memblock. + */ + bootloader_memory_limit = memblock_end_of_DRAM(); memblock_enforce_memory_limit(memory_limit); + memblock_add(__pa_symbol(_text), (u64)(_end - _text)); + } + + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { + extern u16 memstart_offset_seed; + u64 range = linear_region_size - + (memblock_end_of_DRAM() - memblock_start_of_DRAM()); + + /* + * If the size of the linear region exceeds, by a sufficient + * margin, the size of the region that the available physical + * memory spans, randomize the linear region as well. + */ + if (memstart_offset_seed > 0 && range >= ARM64_MEMSTART_ALIGN) { + range = range / ARM64_MEMSTART_ALIGN + 1; + memstart_addr -= ARM64_MEMSTART_ALIGN * + ((range * memstart_offset_seed) >> 16); + } + } /* * Register the kernel text, kernel data, initrd, and initial * pagetables with memblock. */ - memblock_reserve(__pa(_text), _end - _text); + memblock_reserve(__pa_symbol(_text), _end - _text); #ifdef CONFIG_BLK_DEV_INITRD - if (initrd_start) - memblock_reserve(__virt_to_phys(initrd_start), initrd_end - initrd_start); + if (initrd_start) { + memblock_reserve(initrd_start, initrd_end - initrd_start); + + /* the generic initrd code expects virtual addresses */ + initrd_start = __phys_to_virt(initrd_start); + initrd_end = __phys_to_virt(initrd_end); + } #endif early_init_fdt_scan_reserved_mem(); @@ -302,35 +379,38 @@ #ifdef CONFIG_KASAN " kasan : 0x%16lx - 0x%16lx (%6ld GB)\n" #endif + " modules : 0x%16lx - 0x%16lx (%6ld MB)\n" " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n" + " .init : 0x%p" " - 0x%p" " (%6ld KB)\n" + " .text : 0x%p" " - 0x%p" " (%6ld KB)\n" + " .rodata : 0x%p" " - 0x%p" " (%6ld KB)\n" + " .data : 0x%p" " - 0x%p" " (%6ld KB)\n" #ifdef CONFIG_SPARSEMEM_VMEMMAP " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" " 0x%16lx - 0x%16lx (%6ld MB actual)\n" #endif " fixed : 0x%16lx - 0x%16lx (%6ld KB)\n" " PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n" - " modules : 0x%16lx - 0x%16lx (%6ld MB)\n" - " memory : 0x%16lx - 0x%16lx (%6ld MB)\n" - " .init : 0x%p" " - 0x%p" " (%6ld KB)\n" - " .text : 0x%p" " - 0x%p" " (%6ld KB)\n" - " .data : 0x%p" " - 0x%p" " (%6ld KB)\n", + " memory : 0x%16lx - 0x%16lx (%6ld MB)\n", #ifdef CONFIG_KASAN MLG(KASAN_SHADOW_START, KASAN_SHADOW_END), #endif + MLM(MODULES_VADDR, MODULES_END), MLG(VMALLOC_START, VMALLOC_END), + MLK_ROUNDUP(__init_begin, __init_end), + MLK_ROUNDUP(_text, _etext), + MLK_ROUNDUP(__start_rodata, __init_begin), + MLK_ROUNDUP(_sdata, _edata), #ifdef CONFIG_SPARSEMEM_VMEMMAP MLG(VMEMMAP_START, VMEMMAP_START + VMEMMAP_SIZE), - MLM((unsigned long)virt_to_page(PAGE_OFFSET), + MLM((unsigned long)phys_to_page(memblock_start_of_DRAM()), (unsigned long)virt_to_page(high_memory)), #endif MLK(FIXADDR_START, FIXADDR_TOP), MLM(PCI_IO_START, PCI_IO_END), - MLM(MODULES_VADDR, MODULES_END), - MLM(PAGE_OFFSET, (unsigned long)high_memory), - MLK_ROUNDUP(__init_begin, __init_end), - MLK_ROUNDUP(_text, _etext), - MLK_ROUNDUP(_sdata, _edata)); + MLM(__phys_to_virt(memblock_start_of_DRAM()), + (unsigned long)high_memory)); #undef MLK #undef MLM @@ -356,11 +436,15 @@ } } +static inline void poison_init_mem(void *s, size_t count) +{ + memset(s, 0, count); +} + void free_initmem(void) { - fixup_init(); free_initmem_default(0); - free_alternatives_memory(); + fixup_init(); } #ifdef CONFIG_BLK_DEV_INITRD @@ -381,3 +465,174 @@ __setup("keepinitrd", keepinitrd_setup); #endif + +#ifdef CONFIG_KERNEL_TEXT_RDONLY +void set_kernel_text_ro(void) +{ + unsigned long start = PFN_ALIGN(_stext); + unsigned long end = PFN_ALIGN(_etext); + + /* + * Set the kernel identity mapping for text RO. + */ + set_memory_ro(start, (end - start) >> PAGE_SHIFT); +} +#endif +/* + * Dump out memory limit information on panic. + */ +static int dump_mem_limit(struct notifier_block *self, unsigned long v, void *p) +{ + if (memory_limit != (phys_addr_t)ULLONG_MAX) { + pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20); + } else { + pr_emerg("Memory Limit: none\n"); + } + return 0; +} + +static struct notifier_block mem_limit_notifier = { + .notifier_call = dump_mem_limit, +}; + +static int __init register_mem_limit_dumper(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &mem_limit_notifier); + return 0; +} +__initcall(register_mem_limit_dumper); + +#ifdef CONFIG_MEMORY_HOTPLUG +int arch_add_memory(int nid, u64 start, u64 size, bool for_device) +{ + pg_data_t *pgdat; + struct zone *zone; + unsigned long start_pfn = start >> PAGE_SHIFT; + unsigned long nr_pages = size >> PAGE_SHIFT; + unsigned long end_pfn = start_pfn + nr_pages; + unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); + int ret; + + if (end_pfn > max_sparsemem_pfn) { + pr_err("end_pfn too big"); + return -1; + } + hotplug_paging(start, size); + + /* + * Mark the first page in the range as unusable. This is needed + * because __add_section (within __add_pages) wants pfn_valid + * of it to be false, and in arm64 pfn falid is implemented by + * just checking at the nomap flag for existing blocks. + * + * A small trick here is that __add_section() requires only + * phys_start_pfn (that is the first pfn of a section) to be + * invalid. Regardless of whether it was assumed (by the function + * author) that all pfns within a section are either all valid + * or all invalid, it allows to avoid looping twice (once here, + * second when memblock_clear_nomap() is called) through all + * pfns of the section and modify only one pfn. Thanks to that, + * further, in __add_zone() only this very first pfn is skipped + * and corresponding page is not flagged reserved. Therefore it + * is enough to correct this setup only for it. + * + * When arch_add_memory() returns the walk_memory_range() function + * is called and passed with online_memory_block() callback, + * which execution finally reaches the memory_block_action() + * function, where also only the first pfn of a memory block is + * checked to be reserved. Above, it was first pfn of a section, + * here it is a block but + * (drivers/base/memory.c): + * sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE; + * (include/linux/memory.h): + * #define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS) + * so we can consider block and section equivalently + */ + memblock_mark_nomap(start, 1<node_zones + + zone_for_memory(nid, start, size, ZONE_NORMAL, for_device); + ret = __add_pages(nid, zone, start_pfn, nr_pages); + + /* + * Make the pages usable after they have been added. + * This will make pfn_valid return true + */ + memblock_clear_nomap(start, 1<> PAGE_SHIFT; + unsigned long nr_pages = size >> PAGE_SHIFT; + struct page *page = pfn_to_page(start_pfn); + struct zone *zone; + int ret = 0; + + zone = page_zone(page); + ret = __remove_pages(zone, start_pfn, nr_pages); + WARN_ON_ONCE(ret); + + kernel_physical_mapping_remove(start, start + size); + + return ret; +} + +#endif /* CONFIG_MEMORY_HOTREMOVE */ +static int arm64_online_page(struct page *page) +{ + unsigned long target_pfn = page_to_pfn(page); + unsigned long limit = __phys_to_pfn(bootloader_memory_limit); + + if (target_pfn >= limit) + return -EINVAL; + + __online_page_set_limits(page); + __online_page_increment_counters(page); + __online_page_free(page); + + return 0; +} + +static int __init arm64_memory_hotplug_init(void) +{ + set_online_page_callback(&arm64_online_page); + return 0; +} +subsys_initcall(arm64_memory_hotplug_init); +#endif /* CONFIG_MEMORY_HOTPLUG */ diff -ruw linux-4.4.115/arch/arm64/mm/mmap.c linux-4.4.115-fbx/arch/arm64/mm/mmap.c --- linux-4.4.115/arch/arm64/mm/mmap.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/mmap.c 2019-01-22 16:16:21.563228839 +0100 @@ -51,8 +51,12 @@ { unsigned long rnd; - rnd = (unsigned long)get_random_int() & STACK_RND_MASK; - +#ifdef CONFIG_COMPAT + if (test_thread_flag(TIF_32BIT)) + rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1); + else +#endif + rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1); return rnd << PAGE_SHIFT; } diff -ruw linux-4.4.115/arch/arm64/mm/mmu.c linux-4.4.115-fbx/arch/arm64/mm/mmu.c --- linux-4.4.115/arch/arm64/mm/mmu.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/mmu.c 2019-10-29 09:26:23.013197013 +0100 @@ -29,9 +29,14 @@ #include #include #include +#include +#include +#include +#include #include #include +#include #include #include #include @@ -44,13 +49,22 @@ u64 idmap_t0sz = TCR_T0SZ(VA_BITS); +u64 kimage_voffset __read_mostly; +EXPORT_SYMBOL(kimage_voffset); + /* * Empty_zero_page is a special page that is used for zero-initialized data * and COW. */ -struct page *empty_zero_page; +unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; EXPORT_SYMBOL(empty_zero_page); +static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; +static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; + +static bool dma_overlap(phys_addr_t start, phys_addr_t end); + pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot) { @@ -62,16 +76,30 @@ } EXPORT_SYMBOL(phys_mem_access_prot); -static void __init *early_alloc(unsigned long sz) +static phys_addr_t __init early_pgtable_alloc(void) { phys_addr_t phys; void *ptr; - phys = memblock_alloc(sz, sz); + phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE); BUG_ON(!phys); - ptr = __va(phys); - memset(ptr, 0, sz); - return ptr; + + /* + * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE + * slot will be free, so we can (ab)use the FIX_PTE slot to initialise + * any level of table. + */ + ptr = pte_set_fixmap(phys); + + memset(ptr, 0, PAGE_SIZE); + + /* + * Implicit barriers also ensure the zeroed page is visible to the page + * table walker + */ + pte_clear_fixmap(); + + return phys; } /* @@ -95,24 +123,30 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot, - void *(*alloc)(unsigned long size)) + phys_addr_t (*pgtable_alloc)(void)) { pte_t *pte; if (pmd_none(*pmd) || pmd_sect(*pmd)) { - pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); + phys_addr_t pte_phys; + BUG_ON(!pgtable_alloc); + pte_phys = pgtable_alloc(); + pte = pte_set_fixmap(pte_phys); if (pmd_sect(*pmd)) split_pmd(pmd, pte); - __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); + __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); flush_tlb_all(); + pte_clear_fixmap(); } BUG_ON(pmd_bad(*pmd)); - pte = pte_offset_kernel(pmd, addr); + pte = pte_set_fixmap_offset(pmd, addr); do { set_pte(pte, pfn_pte(pfn, prot)); pfn++; } while (pte++, addr += PAGE_SIZE, addr != end); + + pte_clear_fixmap(); } static void split_pud(pud_t *old_pud, pmd_t *pmd) @@ -127,10 +161,29 @@ } while (pmd++, i++, i < PTRS_PER_PMD); } -static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, - unsigned long addr, unsigned long end, +#ifdef CONFIG_DEBUG_PAGEALLOC +static bool block_mappings_allowed(phys_addr_t (*pgtable_alloc)(void)) +{ + + /* + * If debug_page_alloc is enabled we must map the linear map + * using pages. However, other mappings created by + * create_mapping_noalloc must use sections in some cases. Allow + * sections to be used in those cases, where no pgtable_alloc + * function is provided. + */ + return !pgtable_alloc || !debug_pagealloc_enabled(); +} +#else +static bool block_mappings_allowed(phys_addr_t (*pgtable_alloc)(void)) +{ + return true; +} +#endif + +static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, - void *(*alloc)(unsigned long size)) + phys_addr_t (*pgtable_alloc)(void)) { pmd_t *pmd; unsigned long next; @@ -139,7 +192,10 @@ * Check for initial section mappings in the pgd/pud and remove them. */ if (pud_none(*pud) || pud_sect(*pud)) { - pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t)); + phys_addr_t pmd_phys; + BUG_ON(!pgtable_alloc); + pmd_phys = pgtable_alloc(); + pmd = pmd_set_fixmap(pmd_phys); if (pud_sect(*pud)) { /* * need to have the 1G of mappings continue to be @@ -147,19 +203,21 @@ */ split_pud(pud, pmd); } - pud_populate(mm, pud, pmd); + __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); flush_tlb_all(); + pmd_clear_fixmap(); } BUG_ON(pud_bad(*pud)); - pmd = pmd_offset(pud, addr); + pmd = pmd_set_fixmap_offset(pud, addr); do { next = pmd_addr_end(addr, end); /* try section mapping first */ - if (((addr | next | phys) & ~SECTION_MASK) == 0) { + if (((addr | next | phys) & ~SECTION_MASK) == 0 && + block_mappings_allowed(pgtable_alloc) && + !dma_overlap(phys, phys + next - addr)) { pmd_t old_pmd =*pmd; - set_pmd(pmd, __pmd(phys | - pgprot_val(mk_sect_prot(prot)))); + pmd_set_huge(pmd, phys, prot); /* * Check for previous table entries created during * boot (__create_page_tables) and flush them. @@ -167,17 +225,19 @@ if (!pmd_none(old_pmd)) { flush_tlb_all(); if (pmd_table(old_pmd)) { - phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0)); + phys_addr_t table = pmd_page_paddr(old_pmd); if (!WARN_ON_ONCE(slab_is_available())) memblock_free(table, PAGE_SIZE); } } } else { alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), - prot, alloc); + prot, pgtable_alloc); } phys += next - addr; } while (pmd++, addr = next, addr != end); + + pmd_clear_fixmap(); } static inline bool use_1G_block(unsigned long addr, unsigned long next, @@ -192,31 +252,33 @@ return true; } -static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, - unsigned long addr, unsigned long end, +static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, - void *(*alloc)(unsigned long size)) + phys_addr_t (*pgtable_alloc)(void)) { pud_t *pud; unsigned long next; if (pgd_none(*pgd)) { - pud = alloc(PTRS_PER_PUD * sizeof(pud_t)); - pgd_populate(mm, pgd, pud); + phys_addr_t pud_phys; + BUG_ON(!pgtable_alloc); + pud_phys = pgtable_alloc(); + __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); } BUG_ON(pgd_bad(*pgd)); - pud = pud_offset(pgd, addr); + pud = pud_set_fixmap_offset(pgd, addr); do { next = pud_addr_end(addr, end); /* * For 4K granule only, attempt to put down a 1GB block */ - if (use_1G_block(addr, next, phys)) { + if (use_1G_block(addr, next, phys) && + block_mappings_allowed(pgtable_alloc) && + !dma_overlap(phys, phys + next - addr)) { pud_t old_pud = *pud; - set_pud(pud, __pud(phys | - pgprot_val(mk_sect_prot(prot)))); + pud_set_huge(pud, phys, prot); /* * If we have an old value for a pud, it will @@ -228,51 +290,74 @@ if (!pud_none(old_pud)) { flush_tlb_all(); if (pud_table(old_pud)) { - phys_addr_t table = __pa(pmd_offset(&old_pud, 0)); + phys_addr_t table = pud_page_paddr(old_pud); if (!WARN_ON_ONCE(slab_is_available())) memblock_free(table, PAGE_SIZE); } } } else { - alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc); + alloc_init_pmd(pud, addr, next, phys, prot, + pgtable_alloc); } phys += next - addr; } while (pud++, addr = next, addr != end); + + pud_clear_fixmap(); } /* * Create the page directory entries and any necessary page tables for the * mapping specified by 'md'. */ -static void __create_mapping(struct mm_struct *mm, pgd_t *pgd, - phys_addr_t phys, unsigned long virt, +static void init_pgd(pgd_t *pgd, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot, - void *(*alloc)(unsigned long size)) + phys_addr_t (*pgtable_alloc)(void)) { unsigned long addr, length, end, next; + /* + * If the virtual and physical address don't have the same offset + * within a page, we cannot map the region as the caller expects. + */ + if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) + return; + + phys &= PAGE_MASK; addr = virt & PAGE_MASK; length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); end = addr + length; do { next = pgd_addr_end(addr, end); - alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc); + alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc); phys += next - addr; } while (pgd++, addr = next, addr != end); } -static void *late_alloc(unsigned long size) +static phys_addr_t late_pgtable_alloc(void) { - void *ptr; - - BUG_ON(size > PAGE_SIZE); - ptr = (void *)__get_free_page(PGALLOC_GFP); + void *ptr = (void *)__get_free_page(PGALLOC_GFP); BUG_ON(!ptr); - return ptr; + + /* Ensure the zeroed page is visible to the page table walker */ + dsb(ishst); + return __pa(ptr); } -static void __init create_mapping(phys_addr_t phys, unsigned long virt, +static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*alloc)(void)) +{ + init_pgd(pgd_offset_raw(pgdir, virt), phys, virt, size, prot, alloc); +} + +/* + * This function can only be used to modify existing table entries, + * without allocating new levels of table. Note that this permits the + * creation of new section or page entries. + */ +static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot) { if (virt < VMALLOC_START) { @@ -280,16 +365,16 @@ &phys, virt); return; } - __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt, - size, prot, early_alloc); + __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, + NULL); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot) { - __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot, - late_alloc); + __create_pgd_mapping(mm->pgd, phys, virt, size, prot, + late_pgtable_alloc); } static void create_mapping_late(phys_addr_t phys, unsigned long virt, @@ -301,69 +386,57 @@ return; } - return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), - phys, virt, size, prot, late_alloc); + __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, + late_pgtable_alloc); } -#ifdef CONFIG_DEBUG_RODATA -static void __init __map_memblock(phys_addr_t start, phys_addr_t end) +static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) { + unsigned long kernel_start = __pa_symbol(_text); + unsigned long kernel_end = __pa_symbol(__init_begin); + /* - * Set up the executable regions using the existing section mappings - * for now. This will get more fine grained later once all memory - * is mapped + * Take care not to create a writable alias for the + * read-only text and rodata sections of the kernel image. */ - unsigned long kernel_x_start = round_down(__pa(_stext), SWAPPER_BLOCK_SIZE); - unsigned long kernel_x_end = round_up(__pa(__init_end), SWAPPER_BLOCK_SIZE); - if (end < kernel_x_start) { - create_mapping(start, __phys_to_virt(start), - end - start, PAGE_KERNEL); - } else if (start >= kernel_x_end) { - create_mapping(start, __phys_to_virt(start), - end - start, PAGE_KERNEL); - } else { - if (start < kernel_x_start) - create_mapping(start, __phys_to_virt(start), - kernel_x_start - start, - PAGE_KERNEL); - create_mapping(kernel_x_start, - __phys_to_virt(kernel_x_start), - kernel_x_end - kernel_x_start, - PAGE_KERNEL_EXEC); - if (kernel_x_end < end) - create_mapping(kernel_x_end, - __phys_to_virt(kernel_x_end), - end - kernel_x_end, - PAGE_KERNEL); + /* No overlap with the kernel text/rodata */ + if (end < kernel_start || start >= kernel_end) { + __create_pgd_mapping(pgd, start, __phys_to_virt(start), + end - start, PAGE_KERNEL, + early_pgtable_alloc); + return; } + /* + * This block overlaps the kernel text/rodata mappings. + * Map the portion(s) which don't overlap. + */ + if (start < kernel_start) + __create_pgd_mapping(pgd, start, + __phys_to_virt(start), + kernel_start - start, PAGE_KERNEL, + early_pgtable_alloc); + if (kernel_end < end) + __create_pgd_mapping(pgd, kernel_end, + __phys_to_virt(kernel_end), + end - kernel_end, PAGE_KERNEL, + early_pgtable_alloc); + + /* + * Map the linear alias of the [_text, __init_begin) interval as + * read-only/non-executable. This makes the contents of the + * region accessible to subsystems such as hibernate, but + * protects it from inadvertent modification or execution. + */ + __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), + kernel_end - kernel_start, PAGE_KERNEL_RO, + early_pgtable_alloc); } -#else -static void __init __map_memblock(phys_addr_t start, phys_addr_t end) -{ - create_mapping(start, __phys_to_virt(start), end - start, - PAGE_KERNEL_EXEC); -} -#endif -static void __init map_mem(void) +static void __init map_mem(pgd_t *pgd) { struct memblock_region *reg; - phys_addr_t limit; - - /* - * Temporarily limit the memblock range. We need to do this as - * create_mapping requires puds, pmds and ptes to be allocated from - * memory addressable from the initial direct kernel mapping. - * - * The initial direct kernel mapping, located at swapper_pg_dir, gives - * us PUD_SIZE (with SECTION maps) or PMD_SIZE (without SECTION maps, - * memory starting from PHYS_OFFSET (which must be aligned to 2MB as - * per Documentation/arm64/booting.txt). - */ - limit = PHYS_OFFSET + SWAPPER_INIT_MAP_SIZE; - memblock_set_current_limit(limit); /* map all the memory banks */ for_each_memblock(memory, reg) { @@ -372,70 +445,160 @@ if (start >= end) break; + if (memblock_is_nomap(reg)) + continue; + + __map_memblock(pgd, start, end); + } +} + +void mark_rodata_ro(void) +{ + unsigned long section_size; - if (ARM64_SWAPPER_USES_SECTION_MAPS) { + section_size = (unsigned long)_etext - (unsigned long)_text; + create_mapping_late(__pa_symbol(_text), (unsigned long)_text, + section_size, PAGE_KERNEL_ROX); /* - * For the first memory bank align the start address and - * current memblock limit to prevent create_mapping() from - * allocating pte page tables from unmapped memory. With - * the section maps, if the first block doesn't end on section - * size boundary, create_mapping() will try to allocate a pte - * page, which may be returned from an unmapped area. - * When section maps are not used, the pte page table for the - * current limit is already present in swapper_pg_dir. + * mark .rodata as read only. Use __init_begin rather than __end_rodata + * to cover NOTES and EXCEPTION_TABLE. */ - if (start < limit) - start = ALIGN(start, SECTION_SIZE); - if (end < limit) { - limit = end & SECTION_MASK; - memblock_set_current_limit(limit); + section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; + create_mapping_late(__pa_symbol(__start_rodata), + (unsigned long)__start_rodata, + section_size, PAGE_KERNEL_RO); } + +void fixup_init(void) +{ + /* + * Unmap the __init region but leave the VM area in place. This + * prevents the region from being reused for kernel modules, which + * is not supported by kallsyms. + */ + unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin)); } - __map_memblock(start, end); - } - /* Limit no longer required. */ - memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); +static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, + pgprot_t prot, struct vm_struct *vma) +{ + phys_addr_t pa_start = __pa_symbol(va_start); + unsigned long size = va_end - va_start; + + BUG_ON(!PAGE_ALIGNED(pa_start)); + BUG_ON(!PAGE_ALIGNED(size)); + + __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, + early_pgtable_alloc); + + vma->addr = va_start; + vma->phys_addr = pa_start; + vma->size = size; + vma->flags = VM_MAP; + vma->caller = __builtin_return_address(0); + + vm_area_add_early(vma); } -static void __init fixup_executable(void) +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +static int __init map_entry_trampoline(void) { -#ifdef CONFIG_DEBUG_RODATA - /* now that we are actually fully mapped, make the start/end more fine grained */ - if (!IS_ALIGNED((unsigned long)_stext, SWAPPER_BLOCK_SIZE)) { - unsigned long aligned_start = round_down(__pa(_stext), - SWAPPER_BLOCK_SIZE); + extern char __entry_tramp_text_start[]; + + pgprot_t prot = PAGE_KERNEL_EXEC; + phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); + + /* The trampoline is always mapped and can therefore be global */ + pgprot_val(prot) &= ~PTE_NG; + + /* Map only the text into the trampoline page table */ + memset(tramp_pg_dir, 0, PGD_SIZE); + __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE, + prot, late_pgtable_alloc); + + /* Map both the text and data into the kernel page table */ + __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot); + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { + extern char __entry_tramp_data_start[]; - create_mapping(aligned_start, __phys_to_virt(aligned_start), - __pa(_stext) - aligned_start, - PAGE_KERNEL); + __set_fixmap(FIX_ENTRY_TRAMP_DATA, + __pa_symbol(__entry_tramp_data_start), + PAGE_KERNEL_RO); } - if (!IS_ALIGNED((unsigned long)__init_end, SWAPPER_BLOCK_SIZE)) { - unsigned long aligned_end = round_up(__pa(__init_end), - SWAPPER_BLOCK_SIZE); - create_mapping(__pa(__init_end), (unsigned long)__init_end, - aligned_end - __pa(__init_end), - PAGE_KERNEL); + return 0; } +core_initcall(map_entry_trampoline); #endif -} -#ifdef CONFIG_DEBUG_RODATA -void mark_rodata_ro(void) +/* + * Create fine-grained mappings for the kernel. + */ +static void __init map_kernel(pgd_t *pgd) { - create_mapping_late(__pa(_stext), (unsigned long)_stext, - (unsigned long)_etext - (unsigned long)_stext, - PAGE_KERNEL_ROX); + static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data; + + map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text); + map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata); + map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC, + &vmlinux_init); + map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data); + if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) { + /* + * The fixmap falls in a separate pgd to the kernel, and doesn't + * live in the carveout for the swapper_pg_dir. We can simply + * re-use the existing dir for the fixmap. + */ + set_pgd(pgd_offset_raw(pgd, FIXADDR_START), + *pgd_offset_k(FIXADDR_START)); + } else if (CONFIG_PGTABLE_LEVELS > 3) { + /* + * The fixmap shares its top level pgd entry with the kernel + * mapping. This can really only occur when we are running + * with 16k/4 levels, so we can simply reuse the pud level + * entry instead. + */ + BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); + set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START), + __pud(__pa_symbol(bm_pmd) | PUD_TYPE_TABLE)); + pud_clear_fixmap(); + } else { + BUG(); } -#endif -void fixup_init(void) + kasan_copy_shadow(pgd); +} + +struct dma_contig_early_reserve { + phys_addr_t base; + unsigned long size; +}; + +static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS]; + +static int dma_mmu_remap_num; + +void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) { - create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin, - (unsigned long)__init_end - (unsigned long)__init_begin, - PAGE_KERNEL); + dma_mmu_remap[dma_mmu_remap_num].base = base; + dma_mmu_remap[dma_mmu_remap_num].size = size; + dma_mmu_remap_num++; +} + +static bool dma_overlap(phys_addr_t start, phys_addr_t end) +{ + int i; + + for (i = 0; i < dma_mmu_remap_num; i++) { + phys_addr_t dma_base = dma_mmu_remap[i].base; + phys_addr_t dma_end = dma_mmu_remap[i].base + + dma_mmu_remap[i].size; + + if ((dma_base < end) && (dma_end > start)) + return true; + } + return false; } /* @@ -444,31 +607,455 @@ */ void __init paging_init(void) { - void *zero_page; + phys_addr_t pgd_phys = early_pgtable_alloc(); + pgd_t *pgd = pgd_set_fixmap(pgd_phys); + + map_kernel(pgd); + map_mem(pgd); + + /* + * We want to reuse the original swapper_pg_dir so we don't have to + * communicate the new address to non-coherent secondaries in + * secondary_entry, and so cpu_switch_mm can generate the address with + * adrp+add rather than a load from some global variable. + * + * To do this we need to go via a temporary pgd. + */ + cpu_replace_ttbr1(__va(pgd_phys)); + memcpy(swapper_pg_dir, pgd, PAGE_SIZE); + cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); - map_mem(); - fixup_executable(); + pgd_clear_fixmap(); + memblock_free(pgd_phys, PAGE_SIZE); - /* allocate the zero page. */ - zero_page = early_alloc(PAGE_SIZE); + /* + * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd + * allocated with it. + */ + memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE, + SWAPPER_DIR_SIZE - PAGE_SIZE); bootmem_init(); +} - empty_zero_page = virt_to_page(zero_page); +#ifdef CONFIG_MEMORY_HOTPLUG +static phys_addr_t pgd_pgtable_alloc(void) +{ + void *ptr = (void *)__get_free_page(PGALLOC_GFP); + if (!ptr || !pgtable_page_ctor(virt_to_page(ptr))) + BUG(); - /* Ensure the zero page is visible to the page table walker */ + /* Ensure the zeroed page is visible to the page table walker */ dsb(ishst); + return __pa(ptr); +} /* - * TTBR0 is only used for the identity mapping at this stage. Make it - * point to zero page to avoid speculatively fetching new entries. + * hotplug_paging() is used by memory hotplug to build new page tables + * for hot added memory. */ - cpu_set_reserved_ttbr0(); - local_flush_tlb_all(); - cpu_set_default_tcr_t0sz(); +void hotplug_paging(phys_addr_t start, phys_addr_t size) +{ + + struct page *pg; + phys_addr_t pgd_phys = pgd_pgtable_alloc(); + pgd_t *pgd = pgd_set_fixmap(pgd_phys); + + memcpy(pgd, swapper_pg_dir, PAGE_SIZE); + + __create_pgd_mapping(pgd, start, __phys_to_virt(start), size, + PAGE_KERNEL, pgd_pgtable_alloc); + + cpu_replace_ttbr1(__va(pgd_phys)); + memcpy(swapper_pg_dir, pgd, PAGE_SIZE); + cpu_replace_ttbr1(swapper_pg_dir); + + pgd_clear_fixmap(); + + pg = phys_to_page(pgd_phys); + pgtable_page_dtor(pg); + __free_pages(pg, 0); +} + +#ifdef CONFIG_MEMORY_HOTREMOVE +#define PAGE_INUSE 0xFD + +static void free_pagetable(struct page *page, int order, bool direct) +{ + unsigned long magic; + unsigned int nr_pages = 1 << order; + + /* bootmem page has reserved flag */ + if (PageReserved(page)) { + __ClearPageReserved(page); + + magic = (unsigned long)page->lru.next; + if (magic == SECTION_INFO || magic == MIX_SECTION_INFO) { + while (nr_pages--) + put_page_bootmem(page++); + } else { + while (nr_pages--) + free_reserved_page(page++); + } + } else { + /* + * Only direct pagetable allocation (those allocated via + * hotplug) call the pgtable_page_ctor; vmemmap pgtable + * allocations don't. + */ + if (direct) + pgtable_page_dtor(page); + + free_pages((unsigned long)page_address(page), order); + } +} + +static void free_pte_table(pmd_t *pmd, bool direct) +{ + pte_t *pte_start, *pte; + struct page *page; + int i; + + pte_start = (pte_t *) pmd_page_vaddr(*pmd); + /* Check if there is no valid entry in the PMD */ + for (i = 0; i < PTRS_PER_PTE; i++) { + pte = pte_start + i; + if (!pte_none(*pte)) + return; + } + + page = pmd_page(*pmd); + + free_pagetable(page, 0, direct); + + /* + * This spin lock could be only taken in _pte_aloc_kernel + * in mm/memory.c and nowhere else (for arm64). Not sure if + * the function above can be called concurrently. In doubt, + * I am living it here for now, but it probably can be removed + */ + spin_lock(&init_mm.page_table_lock); + pmd_clear(pmd); + spin_unlock(&init_mm.page_table_lock); +} + +static void free_pmd_table(pud_t *pud, bool direct) +{ + pmd_t *pmd_start, *pmd; + struct page *page; + int i; + + pmd_start = (pmd_t *) pud_page_vaddr(*pud); + /* Check if there is no valid entry in the PMD */ + for (i = 0; i < PTRS_PER_PMD; i++) { + pmd = pmd_start + i; + if (!pmd_none(*pmd)) + return; + } + + page = pud_page(*pud); + + free_pagetable(page, 0, direct); + + /* + * This spin lock could be only taken in _pte_aloc_kernel + * in mm/memory.c and nowhere else (for arm64). Not sure if + * the function above can be called concurrently. In doubt, + * I am living it here for now, but it probably can be removed + */ + spin_lock(&init_mm.page_table_lock); + pud_clear(pud); + spin_unlock(&init_mm.page_table_lock); } /* + * When the PUD is folded on the PGD (three levels of paging), + * there's no need to free PUDs + */ +#if CONFIG_PGTABLE_LEVELS > 3 +static void free_pud_table(pgd_t *pgd, bool direct) +{ + pud_t *pud_start, *pud; + struct page *page; + int i; + + pud_start = (pud_t *) pgd_page_vaddr(*pgd); + /* Check if there is no valid entry in the PUD */ + for (i = 0; i < PTRS_PER_PUD; i++) { + pud = pud_start + i; + if (!pud_none(*pud)) + return; + } + + page = pgd_page(*pgd); + + free_pagetable(page, 0, direct); + + /* + * This spin lock could be only + * taken in _pte_aloc_kernel in + * mm/memory.c and nowhere else + * (for arm64). Not sure if the + * function above can be called + * concurrently. In doubt, + * I am living it here for now, + * but it probably can be removed. + */ + spin_lock(&init_mm.page_table_lock); + pgd_clear(pgd); + spin_unlock(&init_mm.page_table_lock); +} +#endif + +static void remove_pte_table(pte_t *pte, unsigned long addr, + unsigned long end, bool direct) +{ + unsigned long next; + void *page_addr; + + for (; addr < end; addr = next, pte++) { + next = (addr + PAGE_SIZE) & PAGE_MASK; + if (next > end) + next = end; + + if (!pte_present(*pte)) + continue; + + if (PAGE_ALIGNED(addr) && PAGE_ALIGNED(next)) { + /* + * Do not free direct mapping pages since they were + * freed when offlining, or simplely not in use. + */ + if (!direct) + free_pagetable(pte_page(*pte), 0, direct); + + /* + * This spin lock could be only + * taken in _pte_aloc_kernel in + * mm/memory.c and nowhere else + * (for arm64). Not sure if the + * function above can be called + * concurrently. In doubt, + * I am living it here for now, + * but it probably can be removed. + */ + spin_lock(&init_mm.page_table_lock); + pte_clear(&init_mm, addr, pte); + spin_unlock(&init_mm.page_table_lock); + } else { + /* + * If we are here, we are freeing vmemmap pages since + * direct mapped memory ranges to be freed are aligned. + * + * If we are not removing the whole page, it means + * other page structs in this page are being used and + * we canot remove them. So fill the unused page_structs + * with 0xFD, and remove the page when it is wholly + * filled with 0xFD. + */ + memset((void *)addr, PAGE_INUSE, next - addr); + + page_addr = page_address(pte_page(*pte)); + if (!memchr_inv(page_addr, PAGE_INUSE, PAGE_SIZE)) { + free_pagetable(pte_page(*pte), 0, direct); + + /* + * This spin lock could be only + * taken in _pte_aloc_kernel in + * mm/memory.c and nowhere else + * (for arm64). Not sure if the + * function above can be called + * concurrently. In doubt, + * I am living it here for now, + * but it probably can be removed. + */ + spin_lock(&init_mm.page_table_lock); + pte_clear(&init_mm, addr, pte); + spin_unlock(&init_mm.page_table_lock); + } + } + } + + // I am adding this flush here in simmetry to the x86 code. + // Why do I need to call it here and not in remove_p[mu]d + flush_tlb_all(); +} + +static void remove_pmd_table(pmd_t *pmd, unsigned long addr, + unsigned long end, bool direct) +{ + unsigned long next; + void *page_addr; + pte_t *pte; + + for (; addr < end; addr = next, pmd++) { + next = pmd_addr_end(addr, end); + + if (!pmd_present(*pmd)) + continue; + + // check if we are using 2MB section mappings + if (pmd_sect(*pmd)) { + if (PAGE_ALIGNED(addr) && PAGE_ALIGNED(next)) { + if (!direct) { + free_pagetable(pmd_page(*pmd), + get_order(PMD_SIZE), direct); + } + /* + * This spin lock could be only + * taken in _pte_aloc_kernel in + * mm/memory.c and nowhere else + * (for arm64). Not sure if the + * function above can be called + * concurrently. In doubt, + * I am living it here for now, + * but it probably can be removed. + */ + spin_lock(&init_mm.page_table_lock); + pmd_clear(pmd); + spin_unlock(&init_mm.page_table_lock); + } else { + /* If here, we are freeing vmemmap pages. */ + memset((void *)addr, PAGE_INUSE, next - addr); + + page_addr = page_address(pmd_page(*pmd)); + if (!memchr_inv(page_addr, PAGE_INUSE, + PMD_SIZE)) { + free_pagetable(pmd_page(*pmd), + get_order(PMD_SIZE), direct); + + /* + * This spin lock could be only + * taken in _pte_aloc_kernel in + * mm/memory.c and nowhere else + * (for arm64). Not sure if the + * function above can be called + * concurrently. In doubt, + * I am living it here for now, + * but it probably can be removed. + */ + spin_lock(&init_mm.page_table_lock); + pmd_clear(pmd); + spin_unlock(&init_mm.page_table_lock); + } + } + continue; + } + + BUG_ON(!pmd_table(*pmd)); + + pte = pte_offset_map(pmd, addr); + remove_pte_table(pte, addr, next, direct); + free_pte_table(pmd, direct); + } +} + +static void remove_pud_table(pud_t *pud, unsigned long addr, + unsigned long end, bool direct) +{ + unsigned long next; + pmd_t *pmd; + void *page_addr; + + for (; addr < end; addr = next, pud++) { + next = pud_addr_end(addr, end); + if (!pud_present(*pud)) + continue; + /* + * If we are using 4K granules, check if we are using + * 1GB section mapping. + */ + if (pud_sect(*pud)) { + if (PAGE_ALIGNED(addr) && PAGE_ALIGNED(next)) { + if (!direct) { + free_pagetable(pud_page(*pud), + get_order(PUD_SIZE), direct); + } + + /* + * This spin lock could be only + * taken in _pte_aloc_kernel in + * mm/memory.c and nowhere else + * (for arm64). Not sure if the + * function above can be called + * concurrently. In doubt, + * I am living it here for now, + * but it probably can be removed. + */ + spin_lock(&init_mm.page_table_lock); + pud_clear(pud); + spin_unlock(&init_mm.page_table_lock); + } else { + /* If here, we are freeing vmemmap pages. */ + memset((void *)addr, PAGE_INUSE, next - addr); + + page_addr = page_address(pud_page(*pud)); + if (!memchr_inv(page_addr, PAGE_INUSE, + PUD_SIZE)) { + + free_pagetable(pud_page(*pud), + get_order(PUD_SIZE), direct); + + /* + * This spin lock could be only + * taken in _pte_aloc_kernel in + * mm/memory.c and nowhere else + * (for arm64). Not sure if the + * function above can be called + * concurrently. In doubt, + * I am living it here for now, + * but it probably can be removed. + */ + spin_lock(&init_mm.page_table_lock); + pud_clear(pud); + spin_unlock(&init_mm.page_table_lock); + } + } + continue; + } + + BUG_ON(!pud_table(*pud)); + + pmd = pmd_offset(pud, addr); + remove_pmd_table(pmd, addr, next, direct); + free_pmd_table(pud, direct); + } +} + +void remove_pagetable(unsigned long start, unsigned long end, bool direct) +{ + unsigned long next; + unsigned long addr; + pgd_t *pgd; + pud_t *pud; + + for (addr = start; addr < end; addr = next) { + next = pgd_addr_end(addr, end); + + pgd = pgd_offset_k(addr); + if (pgd_none(*pgd)) + continue; + + pud = pud_offset(pgd, addr); + remove_pud_table(pud, addr, next, direct); + /* + * When the PUD is folded on the PGD (three levels of paging), + * I did already clear the PMD page in free_pmd_table, + * and reset the corresponding PGD==PUD entry. + */ +#if CONFIG_PGTABLE_LEVELS > 3 + free_pud_table(pgd, direct); +#endif + } + + flush_tlb_all(); +} + + +#endif /* CONFIG_MEMORY_HOTREMOVE */ +#endif /* CONFIG_MEMORY_HOTPLUG */ + +/* * Check whether a kernel address is valid (derived from arch/x86/). */ int kern_addr_valid(unsigned long addr) @@ -519,6 +1106,7 @@ pgd_t *pgd; pud_t *pud; pmd_t *pmd; + int ret = 0; do { next = pmd_addr_end(addr, end); @@ -536,37 +1124,40 @@ void *p = NULL; p = vmemmap_alloc_block_buf(PMD_SIZE, node); - if (!p) - return -ENOMEM; + if (!p) { +#ifdef CONFIG_MEMORY_HOTPLUG + vmemmap_free(start, end); +#endif + ret = -ENOMEM; + break; + } set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); } else vmemmap_verify((pte_t *)pmd, node, addr, next); } while (addr = next, addr != end); - return 0; + if (ret) + return vmemmap_populate_basepages(start, end, node); + else + return ret; } #endif /* CONFIG_ARM64_64K_PAGES */ void vmemmap_free(unsigned long start, unsigned long end) { +#ifdef CONFIG_MEMORY_HOTREMOVE + remove_pagetable(start, end, false); +#endif } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ -static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; -#if CONFIG_PGTABLE_LEVELS > 2 -static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss; -#endif -#if CONFIG_PGTABLE_LEVELS > 3 -static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss; -#endif - static inline pud_t * fixmap_pud(unsigned long addr) { pgd_t *pgd = pgd_offset_k(addr); BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); - return pud_offset(pgd, addr); + return pud_offset_kimg(pgd, addr); } static inline pmd_t * fixmap_pmd(unsigned long addr) @@ -575,18 +1166,20 @@ BUG_ON(pud_none(*pud) || pud_bad(*pud)); - return pmd_offset(pud, addr); + return pmd_offset_kimg(pud, addr); } static inline pte_t * fixmap_pte(unsigned long addr) { - pmd_t *pmd = fixmap_pmd(addr); - - BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd)); - - return pte_offset_kernel(pmd, addr); + return &bm_pte[pte_index(addr)]; } +/* + * The p*d_populate functions call virt_to_phys implicitly so they can't be used + * directly on kernel symbols (bm_p*d). This function is called too early to use + * lm_alias so __p*d_populate functions must be used to populate with the + * physical address from __pa_symbol. + */ void __init early_fixmap_init(void) { pgd_t *pgd; @@ -595,15 +1188,29 @@ unsigned long addr = FIXADDR_START; pgd = pgd_offset_k(addr); - pgd_populate(&init_mm, pgd, bm_pud); - pud = pud_offset(pgd, addr); - pud_populate(&init_mm, pud, bm_pmd); - pmd = pmd_offset(pud, addr); - pmd_populate_kernel(&init_mm, pmd, bm_pte); + if (CONFIG_PGTABLE_LEVELS > 3 && + !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa_symbol(bm_pud))) { + /* + * We only end up here if the kernel mapping and the fixmap + * share the top level pgd entry, which should only happen on + * 16k/4 levels configurations. + */ + BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); + pud = pud_offset_kimg(pgd, addr); + } else { + if (pgd_none(*pgd)) + __pgd_populate(pgd, __pa_symbol(bm_pud), + PUD_TYPE_TABLE); + pud = fixmap_pud(addr); + } + if (pud_none(*pud)) + __pud_populate(pud, __pa_symbol(bm_pmd), PMD_TYPE_TABLE); + pmd = fixmap_pmd(addr); + __pmd_populate(pmd, __pa_symbol(bm_pte), PMD_TYPE_TABLE); /* * The boot-ioremap range spans multiple pmds, for which - * we are not preparted: + * we are not prepared: */ BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); @@ -642,11 +1249,10 @@ } } -void *__init fixmap_remap_fdt(phys_addr_t dt_phys) +void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) { const u64 dt_virt_base = __fix_to_virt(FIX_FDT); - pgprot_t prot = PAGE_KERNEL_RO; - int size, offset; + int offset; void *dt_virt; /* @@ -663,7 +1269,7 @@ /* * Make sure that the FDT region can be mapped without the need to * allocate additional translation table pages, so that it is safe - * to call create_mapping() this early. + * to call create_mapping_noalloc() this early. * * On 64k pages, the FDT will be mapped using PTEs, so we need to * be in the same PMD as the rest of the fixmap. @@ -679,21 +1285,73 @@ dt_virt = (void *)dt_virt_base + offset; /* map the first chunk so we can read the size from the header */ - create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, - SWAPPER_BLOCK_SIZE, prot); + create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), + dt_virt_base, SWAPPER_BLOCK_SIZE, prot); if (fdt_magic(dt_virt) != FDT_MAGIC) return NULL; - size = fdt_totalsize(dt_virt); - if (size > MAX_FDT_SIZE) + *size = fdt_totalsize(dt_virt); + if (*size > MAX_FDT_SIZE) return NULL; - if (offset + size > SWAPPER_BLOCK_SIZE) - create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, - round_up(offset + size, SWAPPER_BLOCK_SIZE), prot); + if (offset + *size > SWAPPER_BLOCK_SIZE) + create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, + round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); - memblock_reserve(dt_phys, size); + return dt_virt; +} + +void *__init fixmap_remap_fdt(phys_addr_t dt_phys) +{ + void *dt_virt; + int size; + dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO); + if (!dt_virt) + return NULL; + + memblock_reserve(dt_phys, size); return dt_virt; } + +int __init arch_ioremap_pud_supported(void) +{ + /* only 4k granule supports level 1 block mappings */ + return IS_ENABLED(CONFIG_ARM64_4K_PAGES); +} + +int __init arch_ioremap_pmd_supported(void) +{ + return 1; +} + +int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) +{ + BUG_ON(phys & ~PUD_MASK); + set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); + return 1; +} + +int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) +{ + BUG_ON(phys & ~PMD_MASK); + set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); + return 1; +} + +int pud_clear_huge(pud_t *pud) +{ + if (!pud_sect(*pud)) + return 0; + pud_clear(pud); + return 1; +} + +int pmd_clear_huge(pmd_t *pmd) +{ + if (!pmd_sect(*pmd)) + return 0; + pmd_clear(pmd); + return 1; +} diff -ruw linux-4.4.115/arch/arm64/mm/pageattr.c linux-4.4.115-fbx/arch/arm64/mm/pageattr.c --- linux-4.4.115/arch/arm64/mm/pageattr.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/pageattr.c 2019-10-29 09:26:23.013197013 +0100 @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -36,14 +37,32 @@ return 0; } +/* + * This function assumes that the range is mapped with PAGE_SIZE pages. + */ +static int __change_memory_common(unsigned long start, unsigned long size, + pgprot_t set_mask, pgprot_t clear_mask) +{ + struct page_change_data data; + int ret; + + data.set_mask = set_mask; + data.clear_mask = clear_mask; + + ret = apply_to_page_range(&init_mm, start, size, change_page_range, + &data); + + flush_tlb_kernel_range(start, start + size); + return ret; +} + static int change_memory_common(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask) { unsigned long start = addr; unsigned long size = PAGE_SIZE*numpages; unsigned long end = start + size; - int ret; - struct page_change_data data; + struct vm_struct *area; if (!PAGE_ALIGNED(addr)) { start &= PAGE_MASK; @@ -51,23 +70,36 @@ WARN_ON_ONCE(1); } + if (!IS_ENABLED(CONFIG_FORCE_PAGES)) { if (start < MODULES_VADDR || start >= MODULES_END) return -EINVAL; if (end < MODULES_VADDR || end >= MODULES_END) return -EINVAL; + } + /* + * Kernel VA mappings are always live, and splitting live section + * mappings into page mappings may cause TLB conflicts. This means + * we have to ensure that changing the permission bits of the range + * we are operating on does not result in such splitting. + * + * Let's restrict ourselves to mappings created by vmalloc (or vmap). + * Those are guaranteed to consist entirely of page mappings, and + * splitting is never needed. + * + * So check whether the [addr, addr + size) interval is entirely + * covered by precisely one VM area that has the VM_ALLOC flag set. + */ + area = find_vm_area((void *)addr); + if (!area || + end > (unsigned long)area->addr + area->size || + !(area->flags & VM_ALLOC)) + return -EINVAL; if (!numpages) return 0; - data.set_mask = set_mask; - data.clear_mask = clear_mask; - - ret = apply_to_page_range(&init_mm, start, size, change_page_range, - &data); - - flush_tlb_kernel_range(start, end); - return ret; + return __change_memory_common(start, size, set_mask, clear_mask); } int set_memory_ro(unsigned long addr, int numpages) @@ -99,3 +131,19 @@ __pgprot(PTE_PXN)); } EXPORT_SYMBOL_GPL(set_memory_x); + +#ifdef CONFIG_DEBUG_PAGEALLOC +void __kernel_map_pages(struct page *page, int numpages, int enable) +{ + unsigned long addr = (unsigned long) page_address(page); + + if (enable) + __change_memory_common(addr, PAGE_SIZE * numpages, + __pgprot(PTE_VALID), + __pgprot(0)); + else + __change_memory_common(addr, PAGE_SIZE * numpages, + __pgprot(0), + __pgprot(PTE_VALID)); +} +#endif diff -ruw linux-4.4.115/arch/arm64/mm/pgd.c linux-4.4.115-fbx/arch/arm64/mm/pgd.c --- linux-4.4.115/arch/arm64/mm/pgd.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/pgd.c 2019-01-22 16:16:21.563228839 +0100 @@ -46,14 +46,14 @@ kmem_cache_free(pgd_cache, pgd); } -static int __init pgd_cache_init(void) +void __init pgd_cache_init(void) { + if (PGD_SIZE == PAGE_SIZE) + return; + /* * Naturally aligned pgds required by the architecture. */ - if (PGD_SIZE != PAGE_SIZE) pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_SIZE, SLAB_PANIC, NULL); - return 0; } -core_initcall(pgd_cache_init); diff -ruw linux-4.4.115/arch/arm64/mm/proc.S linux-4.4.115-fbx/arch/arm64/mm/proc.S --- linux-4.4.115/arch/arm64/mm/proc.S 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/arm64/mm/proc.S 2019-10-29 09:26:23.013197013 +0100 @@ -23,13 +23,11 @@ #include #include #include -#include #include +#include #include #include -#include "proc-macros.S" - #ifdef CONFIG_ARM64_64K_PAGES #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K #elif defined(CONFIG_ARM64_16K_PAGES) @@ -46,6 +44,52 @@ #define MAIR(attr, mt) ((attr) << ((mt) * 8)) /* + * cpu_cache_off() + * + * Turn the CPU D-cache off. + */ +ENTRY(cpu_cache_off) + mrs x0, sctlr_el1 + bic x0, x0, #1 << 2 // clear SCTLR.C + msr sctlr_el1, x0 + isb + ret +ENDPROC(cpu_cache_off) + +/* + * cpu_reset(loc) + * + * Perform a soft reset of the system. Put the CPU into the same state + * as it would be if it had been reset, and branch to what would be the + * reset vector. It must be executed with the flat identity mapping. + * + * - loc - location to jump to for soft reset + */ + .align 5 +ENTRY(cpu_reset) + mrs x1, sctlr_el1 + bic x1, x1, #1 + msr sctlr_el1, x1 // disable the MMU + isb + ret x0 +ENDPROC(cpu_reset) + +ENTRY(cpu_soft_restart) + /* Save address of cpu_reset() and reset address */ + mov x19, x0 + mov x20, x1 + + /* Turn D-cache off */ + bl cpu_cache_off + + /* Push out all dirty data, and ensure cache is empty */ + bl flush_cache_all + + mov x0, x20 + ret x19 +ENDPROC(cpu_soft_restart) + +/* * cpu_do_idle() * * Idle the processor (wait for interrupt). @@ -66,62 +110,56 @@ mrs x2, tpidr_el0 mrs x3, tpidrro_el0 mrs x4, contextidr_el1 - mrs x5, mair_el1 - mrs x6, cpacr_el1 - mrs x7, ttbr1_el1 - mrs x8, tcr_el1 - mrs x9, vbar_el1 - mrs x10, mdscr_el1 - mrs x11, oslsr_el1 - mrs x12, sctlr_el1 + mrs x5, cpacr_el1 + mrs x6, tcr_el1 + mrs x7, vbar_el1 + mrs x8, mdscr_el1 + mrs x9, oslsr_el1 + mrs x10, sctlr_el1 + mrs x11, tpidr_el1 + mrs x12, sp_el0 stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - str x12, [x0, #80] + stp x4, xzr, [x0, #16] + stp x5, x6, [x0, #32] + stp x7, x8, [x0, #48] + stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] ret ENDPROC(cpu_do_suspend) /** * cpu_do_resume - restore CPU register context * - * x0: Physical address of context pointer - * x1: ttbr0_el1 to be restored - * - * Returns: - * sctlr_el1 value in x0 + * x0: Address of context pointer */ ENTRY(cpu_do_resume) - /* - * Invalidate local tlb entries before turning on MMU - */ - tlbi vmalle1 ldp x2, x3, [x0] ldp x4, x5, [x0, #16] - ldp x6, x7, [x0, #32] - ldp x8, x9, [x0, #48] - ldp x10, x11, [x0, #64] - ldr x12, [x0, #80] + ldp x6, x8, [x0, #32] + ldp x9, x10, [x0, #48] + ldp x11, x12, [x0, #64] + ldp x13, x14, [x0, #80] msr tpidr_el0, x2 msr tpidrro_el0, x3 msr contextidr_el1, x4 - msr mair_el1, x5 msr cpacr_el1, x6 - msr ttbr0_el1, x1 - msr ttbr1_el1, x7 - tcr_set_idmap_t0sz x8, x7 + + /* Don't change t0sz here, mask those bits when restoring */ + mrs x5, tcr_el1 + bfi x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH + msr tcr_el1, x8 msr vbar_el1, x9 msr mdscr_el1, x10 + msr sctlr_el1, x12 + msr tpidr_el1, x13 + msr sp_el0, x14 /* * Restore oslsr_el1 by writing oslar_el1 */ ubfx x11, x11, #1, #1 msr oslar_el1, x11 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 - mov x0, x12 - dsb nsh // Make sure local tlb invalidation completed isb ret ENDPROC(cpu_do_resume) @@ -135,24 +173,46 @@ * - pgd_phys - physical address of new TTB */ ENTRY(cpu_do_switch_mm) + mrs x2, ttbr1_el1 mmid x1, x1 // get mm->context.id - bfi x0, x1, #48, #16 // set the ASID - msr ttbr0_el1, x0 // set TTBR0 +#ifdef CONFIG_ARM64_SW_TTBR0_PAN + bfi x0, x1, #48, #16 // set the ASID field in TTBR0 +#endif + bfi x2, x1, #48, #16 // set the ASID + msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set) isb -alternative_if_not ARM64_WORKAROUND_CAVIUM_27456 - ret - nop - nop - nop -alternative_else - ic iallu - dsb nsh + msr ttbr0_el1, x0 // now update TTBR0 isb - ret -alternative_endif + b post_ttbr_update_workaround // Back to C code... ENDPROC(cpu_do_switch_mm) - .section ".text.init", #alloc, #execinstr + .pushsection ".idmap.text", "ax" +/* + * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd) + * + * This is the low-level counterpart to cpu_replace_ttbr1, and should not be + * called by anything else. It can only be executed from a TTBR0 mapping. + */ +ENTRY(idmap_cpu_replace_ttbr1) + mrs x2, daif + msr daifset, #0xf + + adrp x1, empty_zero_page + msr ttbr1_el1, x1 + isb + + tlbi vmalle1 + dsb nsh + isb + + msr ttbr1_el1, x0 + isb + + msr daif, x2 + + ret +ENDPROC(idmap_cpu_replace_ttbr1) + .popsection /* * __cpu_setup @@ -203,7 +263,7 @@ * both user and kernel. */ ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ - TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 + TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 | TCR_A1 tcr_set_idmap_t0sz x10, x9 /* @@ -243,5 +303,16 @@ */ .type crval, #object crval: +#ifdef CONFIG_ARM64_ICACHE_DISABLE +#define CR_IBIT 0 +#else +#define CR_IBIT 0x1000 +#endif + +#ifdef CONFIG_ARM64_DCACHE_DISABLE +#define CR_CBIT 0 +#else +#define CR_CBIT 0x4 +#endif .word 0xfcffffff // clear - .word 0x34d5d91d // set + .word 0x34d5d91d | CR_IBIT | CR_CBIT // set diff -ruw linux-4.4.115/arch/cris/boot/dts/include/dt-bindings/input/linux-event-codes.h linux-4.4.115-fbx/arch/cris/boot/dts/include/dt-bindings/input/linux-event-codes.h --- linux-4.4.115/arch/cris/boot/dts/include/dt-bindings/input/linux-event-codes.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/cris/boot/dts/include/dt-bindings/input/linux-event-codes.h 2019-10-29 09:26:25.545221791 +0100 @@ -611,6 +611,37 @@ #define KEY_KBDINPUTASSIST_ACCEPT 0x264 #define KEY_KBDINPUTASSIST_CANCEL 0x265 +/* Diagonal movement keys */ +#define KEY_RIGHT_UP 0x266 +#define KEY_RIGHT_DOWN 0x267 +#define KEY_LEFT_UP 0x268 +#define KEY_LEFT_DOWN 0x269 + +#define KEY_ROOT_MENU 0x26a /* Show Device's Root Menu */ +/* Show Top Menu of the Media (e.g. DVD) */ +#define KEY_MEDIA_TOP_MENU 0x26b +#define KEY_NUMERIC_11 0x26c +#define KEY_NUMERIC_12 0x26d +/* + * Toggle Audio Description: refers to an audio service that helps blind and + * visually impaired consumers understand the action in a program. Note: in + * some countries this is referred to as "Video Description". + */ +#define KEY_AUDIO_DESC 0x26e +#define KEY_3D_MODE 0x26f +#define KEY_NEXT_FAVORITE 0x270 +#define KEY_STOP_RECORD 0x271 +#define KEY_PAUSE_RECORD 0x272 +#define KEY_VOD 0x273 /* Video on Demand */ +#define KEY_UNMUTE 0x274 +#define KEY_FASTREVERSE 0x275 +#define KEY_SLOWREVERSE 0x276 +/* + * Control a data application associated with the currently viewed channel, + * e.g. teletext or data broadcast application (MHEG, MHP, HbbTV, etc.) + */ +#define KEY_DATA 0x275 + #define BTN_TRIGGER_HAPPY 0x2c0 #define BTN_TRIGGER_HAPPY1 0x2c0 #define BTN_TRIGGER_HAPPY2 0x2c1 @@ -653,6 +684,18 @@ #define BTN_TRIGGER_HAPPY39 0x2e6 #define BTN_TRIGGER_HAPPY40 0x2e7 +#define KEY_APP_TV 0x2f1 +#define KEY_APP_REPLAY 0x2f2 +#define KEY_APP_VIDEOCLUB 0x2f3 +#define KEY_APP_WHATSON 0x2f4 +#define KEY_APP_RECORDS 0x2f5 +#define KEY_APP_MEDIA 0x2f6 +#define KEY_APP_YOUTUBE 0x2f7 +#define KEY_APP_RADIOS 0x2f8 +#define KEY_APP_CANALVOD 0x2f9 +#define KEY_APP_PIP 0x2fa +#define KEY_APP_NETFLIX 0x2fb + /* We avoid low common keys in module aliases so they don't get huge. */ #define KEY_MIN_INTERESTING KEY_MUTE #define KEY_MAX 0x2ff @@ -749,7 +792,11 @@ #define SW_ROTATE_LOCK 0x0c /* set = rotate locked/disabled */ #define SW_LINEIN_INSERT 0x0d /* set = inserted */ #define SW_MUTE_DEVICE 0x0e /* set = device disabled */ -#define SW_MAX 0x0f +#define SW_HPHL_OVERCURRENT 0x0f /* set = over current on left hph */ +#define SW_HPHR_OVERCURRENT 0x10 /* set = over current on right hph */ +#define SW_MICROPHONE2_INSERT 0x11 /* set = inserted */ +#define SW_UNSUPPORT_INSERT 0x12 /* set = unsupported device inserted */ +#define SW_MAX 0x20 #define SW_CNT (SW_MAX+1) /* diff -ruw linux-4.4.115/arch/Kconfig linux-4.4.115-fbx/arch/Kconfig --- linux-4.4.115/arch/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/Kconfig 2019-10-29 09:26:22.861195525 +0100 @@ -225,8 +225,8 @@ config ARCH_TASK_STRUCT_ALLOCATOR bool -# Select if arch has its private alloc_thread_info() function -config ARCH_THREAD_INFO_ALLOCATOR +# Select if arch has its private alloc_thread_stack() function +config ARCH_THREAD_STACK_ALLOCATOR bool # Select if arch wants to size task_struct dynamically via arch_task_struct_size: @@ -423,6 +423,15 @@ endchoice +config HAVE_ARCH_WITHIN_STACK_FRAMES + bool + help + An architecture should select this if it can walk the kernel stack + frames to determine if an object is part of either the arguments + or local variables (i.e. that it excludes saved return addresses, + and similar) by implementing an inline arch_within_stack_frames(), + which is used by CONFIG_HARDENED_USERCOPY. + config HAVE_CONTEXT_TRACKING bool help @@ -518,6 +527,74 @@ normal C parameter passing, rather than extracting the syscall argument from pt_regs. +config HAVE_ARCH_MMAP_RND_BITS + bool + help + An arch should select this symbol if it supports setting a variable + number of bits for use in establishing the base address for mmap + allocations, has MMU enabled and provides values for both: + - ARCH_MMAP_RND_BITS_MIN + - ARCH_MMAP_RND_BITS_MAX + +config ARCH_MMAP_RND_BITS_MIN + int + +config ARCH_MMAP_RND_BITS_MAX + int + +config ARCH_MMAP_RND_BITS_DEFAULT + int + +config ARCH_MMAP_RND_BITS + int "Number of bits to use for ASLR of mmap base address" if EXPERT + range ARCH_MMAP_RND_BITS_MIN ARCH_MMAP_RND_BITS_MAX + default ARCH_MMAP_RND_BITS_DEFAULT if ARCH_MMAP_RND_BITS_DEFAULT + default ARCH_MMAP_RND_BITS_MIN + depends on HAVE_ARCH_MMAP_RND_BITS + help + This value can be used to select the number of bits to use to + determine the random offset to the base address of vma regions + resulting from mmap allocations. This value will be bounded + by the architecture's minimum and maximum supported values. + + This value can be changed after boot using the + /proc/sys/vm/mmap_rnd_bits tunable + +config HAVE_ARCH_MMAP_RND_COMPAT_BITS + bool + help + An arch should select this symbol if it supports running applications + in compatibility mode, supports setting a variable number of bits for + use in establishing the base address for mmap allocations, has MMU + enabled and provides values for both: + - ARCH_MMAP_RND_COMPAT_BITS_MIN + - ARCH_MMAP_RND_COMPAT_BITS_MAX + +config ARCH_MMAP_RND_COMPAT_BITS_MIN + int + +config ARCH_MMAP_RND_COMPAT_BITS_MAX + int + +config ARCH_MMAP_RND_COMPAT_BITS_DEFAULT + int + +config ARCH_MMAP_RND_COMPAT_BITS + int "Number of bits to use for ASLR of mmap base address for compatible applications" if EXPERT + range ARCH_MMAP_RND_COMPAT_BITS_MIN ARCH_MMAP_RND_COMPAT_BITS_MAX + default ARCH_MMAP_RND_COMPAT_BITS_DEFAULT if ARCH_MMAP_RND_COMPAT_BITS_DEFAULT + default ARCH_MMAP_RND_COMPAT_BITS_MIN + depends on HAVE_ARCH_MMAP_RND_COMPAT_BITS + help + This value can be used to select the number of bits to use to + determine the random offset to the base address of vma regions + resulting from mmap allocations for compatible applications This + value will be bounded by the architecture's minimum and maximum + supported values. + + This value can be changed after boot using the + /proc/sys/vm/mmap_rnd_compat_bits tunable + # # ABI hall of shame # diff -ruw linux-4.4.115/arch/metag/boot/dts/include/dt-bindings/input/linux-event-codes.h linux-4.4.115-fbx/arch/metag/boot/dts/include/dt-bindings/input/linux-event-codes.h --- linux-4.4.115/arch/metag/boot/dts/include/dt-bindings/input/linux-event-codes.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/metag/boot/dts/include/dt-bindings/input/linux-event-codes.h 2019-10-29 09:26:25.545221791 +0100 @@ -611,6 +611,37 @@ #define KEY_KBDINPUTASSIST_ACCEPT 0x264 #define KEY_KBDINPUTASSIST_CANCEL 0x265 +/* Diagonal movement keys */ +#define KEY_RIGHT_UP 0x266 +#define KEY_RIGHT_DOWN 0x267 +#define KEY_LEFT_UP 0x268 +#define KEY_LEFT_DOWN 0x269 + +#define KEY_ROOT_MENU 0x26a /* Show Device's Root Menu */ +/* Show Top Menu of the Media (e.g. DVD) */ +#define KEY_MEDIA_TOP_MENU 0x26b +#define KEY_NUMERIC_11 0x26c +#define KEY_NUMERIC_12 0x26d +/* + * Toggle Audio Description: refers to an audio service that helps blind and + * visually impaired consumers understand the action in a program. Note: in + * some countries this is referred to as "Video Description". + */ +#define KEY_AUDIO_DESC 0x26e +#define KEY_3D_MODE 0x26f +#define KEY_NEXT_FAVORITE 0x270 +#define KEY_STOP_RECORD 0x271 +#define KEY_PAUSE_RECORD 0x272 +#define KEY_VOD 0x273 /* Video on Demand */ +#define KEY_UNMUTE 0x274 +#define KEY_FASTREVERSE 0x275 +#define KEY_SLOWREVERSE 0x276 +/* + * Control a data application associated with the currently viewed channel, + * e.g. teletext or data broadcast application (MHEG, MHP, HbbTV, etc.) + */ +#define KEY_DATA 0x275 + #define BTN_TRIGGER_HAPPY 0x2c0 #define BTN_TRIGGER_HAPPY1 0x2c0 #define BTN_TRIGGER_HAPPY2 0x2c1 @@ -653,6 +684,18 @@ #define BTN_TRIGGER_HAPPY39 0x2e6 #define BTN_TRIGGER_HAPPY40 0x2e7 +#define KEY_APP_TV 0x2f1 +#define KEY_APP_REPLAY 0x2f2 +#define KEY_APP_VIDEOCLUB 0x2f3 +#define KEY_APP_WHATSON 0x2f4 +#define KEY_APP_RECORDS 0x2f5 +#define KEY_APP_MEDIA 0x2f6 +#define KEY_APP_YOUTUBE 0x2f7 +#define KEY_APP_RADIOS 0x2f8 +#define KEY_APP_CANALVOD 0x2f9 +#define KEY_APP_PIP 0x2fa +#define KEY_APP_NETFLIX 0x2fb + /* We avoid low common keys in module aliases so they don't get huge. */ #define KEY_MIN_INTERESTING KEY_MUTE #define KEY_MAX 0x2ff @@ -749,7 +792,11 @@ #define SW_ROTATE_LOCK 0x0c /* set = rotate locked/disabled */ #define SW_LINEIN_INSERT 0x0d /* set = inserted */ #define SW_MUTE_DEVICE 0x0e /* set = device disabled */ -#define SW_MAX 0x0f +#define SW_HPHL_OVERCURRENT 0x0f /* set = over current on left hph */ +#define SW_HPHR_OVERCURRENT 0x10 /* set = over current on right hph */ +#define SW_MICROPHONE2_INSERT 0x11 /* set = inserted */ +#define SW_UNSUPPORT_INSERT 0x12 /* set = unsupported device inserted */ +#define SW_MAX 0x20 #define SW_CNT (SW_MAX+1) /* diff -ruw linux-4.4.115/arch/mips/boot/dts/include/dt-bindings/input/linux-event-codes.h linux-4.4.115-fbx/arch/mips/boot/dts/include/dt-bindings/input/linux-event-codes.h --- linux-4.4.115/arch/mips/boot/dts/include/dt-bindings/input/linux-event-codes.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/mips/boot/dts/include/dt-bindings/input/linux-event-codes.h 2019-10-29 09:26:25.545221791 +0100 @@ -611,6 +611,37 @@ #define KEY_KBDINPUTASSIST_ACCEPT 0x264 #define KEY_KBDINPUTASSIST_CANCEL 0x265 +/* Diagonal movement keys */ +#define KEY_RIGHT_UP 0x266 +#define KEY_RIGHT_DOWN 0x267 +#define KEY_LEFT_UP 0x268 +#define KEY_LEFT_DOWN 0x269 + +#define KEY_ROOT_MENU 0x26a /* Show Device's Root Menu */ +/* Show Top Menu of the Media (e.g. DVD) */ +#define KEY_MEDIA_TOP_MENU 0x26b +#define KEY_NUMERIC_11 0x26c +#define KEY_NUMERIC_12 0x26d +/* + * Toggle Audio Description: refers to an audio service that helps blind and + * visually impaired consumers understand the action in a program. Note: in + * some countries this is referred to as "Video Description". + */ +#define KEY_AUDIO_DESC 0x26e +#define KEY_3D_MODE 0x26f +#define KEY_NEXT_FAVORITE 0x270 +#define KEY_STOP_RECORD 0x271 +#define KEY_PAUSE_RECORD 0x272 +#define KEY_VOD 0x273 /* Video on Demand */ +#define KEY_UNMUTE 0x274 +#define KEY_FASTREVERSE 0x275 +#define KEY_SLOWREVERSE 0x276 +/* + * Control a data application associated with the currently viewed channel, + * e.g. teletext or data broadcast application (MHEG, MHP, HbbTV, etc.) + */ +#define KEY_DATA 0x275 + #define BTN_TRIGGER_HAPPY 0x2c0 #define BTN_TRIGGER_HAPPY1 0x2c0 #define BTN_TRIGGER_HAPPY2 0x2c1 @@ -653,6 +684,18 @@ #define BTN_TRIGGER_HAPPY39 0x2e6 #define BTN_TRIGGER_HAPPY40 0x2e7 +#define KEY_APP_TV 0x2f1 +#define KEY_APP_REPLAY 0x2f2 +#define KEY_APP_VIDEOCLUB 0x2f3 +#define KEY_APP_WHATSON 0x2f4 +#define KEY_APP_RECORDS 0x2f5 +#define KEY_APP_MEDIA 0x2f6 +#define KEY_APP_YOUTUBE 0x2f7 +#define KEY_APP_RADIOS 0x2f8 +#define KEY_APP_CANALVOD 0x2f9 +#define KEY_APP_PIP 0x2fa +#define KEY_APP_NETFLIX 0x2fb + /* We avoid low common keys in module aliases so they don't get huge. */ #define KEY_MIN_INTERESTING KEY_MUTE #define KEY_MAX 0x2ff @@ -749,7 +792,11 @@ #define SW_ROTATE_LOCK 0x0c /* set = rotate locked/disabled */ #define SW_LINEIN_INSERT 0x0d /* set = inserted */ #define SW_MUTE_DEVICE 0x0e /* set = device disabled */ -#define SW_MAX 0x0f +#define SW_HPHL_OVERCURRENT 0x0f /* set = over current on left hph */ +#define SW_HPHR_OVERCURRENT 0x10 /* set = over current on right hph */ +#define SW_MICROPHONE2_INSERT 0x11 /* set = inserted */ +#define SW_UNSUPPORT_INSERT 0x12 /* set = unsupported device inserted */ +#define SW_MAX 0x20 #define SW_CNT (SW_MAX+1) /* diff -ruw linux-4.4.115/arch/powerpc/boot/dts/include/dt-bindings/input/linux-event-codes.h linux-4.4.115-fbx/arch/powerpc/boot/dts/include/dt-bindings/input/linux-event-codes.h --- linux-4.4.115/arch/powerpc/boot/dts/include/dt-bindings/input/linux-event-codes.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/arch/powerpc/boot/dts/include/dt-bindings/input/linux-event-codes.h 2019-10-29 09:26:25.545221791 +0100 @@ -611,6 +611,37 @@ #define KEY_KBDINPUTASSIST_ACCEPT 0x264 #define KEY_KBDINPUTASSIST_CANCEL 0x265 +/* Diagonal movement keys */ +#define KEY_RIGHT_UP 0x266 +#define KEY_RIGHT_DOWN 0x267 +#define KEY_LEFT_UP 0x268 +#define KEY_LEFT_DOWN 0x269 + +#define KEY_ROOT_MENU 0x26a /* Show Device's Root Menu */ +/* Show Top Menu of the Media (e.g. DVD) */ +#define KEY_MEDIA_TOP_MENU 0x26b +#define KEY_NUMERIC_11 0x26c +#define KEY_NUMERIC_12 0x26d +/* + * Toggle Audio Description: refers to an audio service that helps blind and + * visually impaired consumers understand the action in a program. Note: in + * some countries this is referred to as "Video Description". + */ +#define KEY_AUDIO_DESC 0x26e +#define KEY_3D_MODE 0x26f +#define KEY_NEXT_FAVORITE 0x270 +#define KEY_STOP_RECORD 0x271 +#define KEY_PAUSE_RECORD 0x272 +#define KEY_VOD 0x273 /* Video on Demand */ +#define KEY_UNMUTE 0x274 +#define KEY_FASTREVERSE 0x275 +#define KEY_SLOWREVERSE 0x276 +/* + * Control a data application associated with the currently viewed channel, + * e.g. teletext or data broadcast application (MHEG, MHP, HbbTV, etc.) + */ +#define KEY_DATA 0x275 + #define BTN_TRIGGER_HAPPY 0x2c0 #define BTN_TRIGGER_HAPPY1 0x2c0 #define BTN_TRIGGER_HAPPY2 0x2c1 @@ -653,6 +684,18 @@ #define BTN_TRIGGER_HAPPY39 0x2e6 #define BTN_TRIGGER_HAPPY40 0x2e7 +#define KEY_APP_TV 0x2f1 +#define KEY_APP_REPLAY 0x2f2 +#define KEY_APP_VIDEOCLUB 0x2f3 +#define KEY_APP_WHATSON 0x2f4 +#define KEY_APP_RECORDS 0x2f5 +#define KEY_APP_MEDIA 0x2f6 +#define KEY_APP_YOUTUBE 0x2f7 +#define KEY_APP_RADIOS 0x2f8 +#define KEY_APP_CANALVOD 0x2f9 +#define KEY_APP_PIP 0x2fa +#define KEY_APP_NETFLIX 0x2fb + /* We avoid low common keys in module aliases so they don't get huge. */ #define KEY_MIN_INTERESTING KEY_MUTE #define KEY_MAX 0x2ff @@ -749,7 +792,11 @@ #define SW_ROTATE_LOCK 0x0c /* set = rotate locked/disabled */ #define SW_LINEIN_INSERT 0x0d /* set = inserted */ #define SW_MUTE_DEVICE 0x0e /* set = device disabled */ -#define SW_MAX 0x0f +#define SW_HPHL_OVERCURRENT 0x0f /* set = over current on left hph */ +#define SW_HPHR_OVERCURRENT 0x10 /* set = over current on right hph */ +#define SW_MICROPHONE2_INSERT 0x11 /* set = inserted */ +#define SW_UNSUPPORT_INSERT 0x12 /* set = unsupported device inserted */ +#define SW_MAX 0x20 #define SW_CNT (SW_MAX+1) /* diff -ruw linux-4.4.115/block/bio.c linux-4.4.115-fbx/block/bio.c --- linux-4.4.115/block/bio.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/bio.c 2019-10-29 09:26:23.329200105 +0100 @@ -31,6 +31,8 @@ #include +#include "blk.h" + /* * Test patch to inline a certain number of bi_io_vec's inside the bio * itself, to shrink a bio data allocation from two mempool calls to one @@ -590,6 +592,7 @@ bio->bi_rw = bio_src->bi_rw; bio->bi_iter = bio_src->bi_iter; bio->bi_io_vec = bio_src->bi_io_vec; + bio->bi_dio_inode = bio_src->bi_dio_inode; bio_clone_blkcg_association(bio, bio_src); } @@ -1782,8 +1785,10 @@ bio_put(bio); bio = parent; } else { - if (bio->bi_end_io) + if (bio->bi_end_io) { + blk_update_perf_stats(bio); bio->bi_end_io(bio); + } bio = NULL; } } diff -ruw linux-4.4.115/block/blk-core.c linux-4.4.115-fbx/block/blk-core.c --- linux-4.4.115/block/blk-core.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-core.c 2019-10-29 09:26:23.333200144 +0100 @@ -11,6 +11,12 @@ /* * This handles all read/write requests to block devices */ + +#ifdef CONFIG_BLOCK_PERF_FRAMEWORK +#define DRIVER_NAME "Block" +#define pr_fmt(fmt) DRIVER_NAME ": %s: " fmt, __func__ +#endif + #include #include #include @@ -34,12 +40,20 @@ #include #include +#ifdef CONFIG_BLOCK_PERF_FRAMEWORK +#include +#include +#include +#endif + #define CREATE_TRACE_POINTS #include #include "blk.h" #include "blk-mq.h" +#include + EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); @@ -73,7 +87,7 @@ * flip its congestion state for events on other blkcgs. */ if (rl == &rl->q->root_rl) - clear_wb_congested(rl->q->backing_dev_info.wb.congested, sync); + clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync); #endif } @@ -84,7 +98,7 @@ #else /* see blk_clear_congested() */ if (rl == &rl->q->root_rl) - set_wb_congested(rl->q->backing_dev_info.wb.congested, sync); + set_wb_congested(rl->q->backing_dev_info->wb.congested, sync); #endif } @@ -108,14 +122,12 @@ * @bdev: device * * Locates the passed device's request queue and returns the address of its - * backing_dev_info. This function can only be called if @bdev is opened - * and the return value is never NULL. + * backing_dev_info. The return value is never NULL however we may return + * &noop_backing_dev_info if the bdev is not currently open. */ struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev) { - struct request_queue *q = bdev_get_queue(bdev); - - return &q->backing_dev_info; + return bdev->bd_bdi; } EXPORT_SYMBOL(blk_get_backing_dev_info); @@ -583,7 +595,7 @@ blk_flush_integrity(); /* @q won't process any more request, flush async actions */ - del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer); + del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer); blk_sync_queue(q); if (q->mq_ops) @@ -595,8 +607,6 @@ q->queue_lock = &q->__queue_lock; spin_unlock_irq(lock); - bdi_unregister(&q->backing_dev_info); - /* @q is and will stay empty, shutdown and put */ blk_put_queue(q); } @@ -685,7 +695,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) { struct request_queue *q; - int err; q = kmem_cache_alloc_node(blk_requestq_cachep, gfp_mask | __GFP_ZERO, node_id); @@ -700,17 +709,17 @@ if (!q->bio_split) goto fail_id; - q->backing_dev_info.ra_pages = + q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id); + if (!q->backing_dev_info) + goto fail_split; + + q->backing_dev_info->ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; - q->backing_dev_info.capabilities = BDI_CAP_CGROUP_WRITEBACK; - q->backing_dev_info.name = "block"; + q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK; + q->backing_dev_info->name = "block"; q->node = node_id; - err = bdi_init(&q->backing_dev_info); - if (err) - goto fail_split; - - setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, + setup_timer(&q->backing_dev_info->laptop_mode_wb_timer, laptop_mode_timer_fn, (unsigned long) q); setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); INIT_LIST_HEAD(&q->queue_head); @@ -760,7 +769,7 @@ fail_ref: percpu_ref_exit(&q->q_usage_counter); fail_bdi: - bdi_destroy(&q->backing_dev_info); + bdi_put(q->backing_dev_info); fail_split: bioset_free(q->bio_split); fail_id: @@ -1183,7 +1192,7 @@ * disturb iosched and blkcg but weird is bettern than dead. */ printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n", - __func__, dev_name(q->backing_dev_info.dev)); + __func__, dev_name(q->backing_dev_info->dev)); rq->cmd_flags &= ~REQ_ELVPRIV; rq->elv.icq = NULL; @@ -1277,8 +1286,6 @@ { struct request *rq; - BUG_ON(rw != READ && rw != WRITE); - /* create ioc upfront */ create_io_context(gfp_mask, q->node); @@ -1450,9 +1457,11 @@ #ifdef CONFIG_PM static void blk_pm_put_request(struct request *rq) { - if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && !--rq->q->nr_pending) + if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && rq->q->nr_pending) { + if (!--rq->q->nr_pending) pm_runtime_mark_last_busy(rq->q->dev); } +} #else static inline void blk_pm_put_request(struct request *rq) {} #endif @@ -1477,6 +1486,9 @@ /* this is a bio leak */ WARN_ON(req->bio != NULL); + /* this is a bio leak if the bio is not tagged with BIO_DONTFREE */ + WARN_ON(req->bio && !bio_flagged(req->bio, BIO_DONTFREE)); + /* * Request may not have originated from ll_rw_blk. if not, * it didn't come out of our reserved rq pools @@ -1698,6 +1710,7 @@ req->ioprio = bio_prio(bio); blk_rq_bio_prep(req->q, req, bio); } +EXPORT_SYMBOL(init_request_from_bio); static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) { @@ -1722,7 +1735,8 @@ return BLK_QC_T_NONE; } - if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { + if (bio->bi_rw & (REQ_FLUSH | REQ_FUA | REQ_POST_FLUSH_BARRIER | + REQ_BARRIER)) { spin_lock_irq(q->queue_lock); where = ELEVATOR_INSERT_FLUSH; goto get_rq; @@ -2105,6 +2119,477 @@ } EXPORT_SYMBOL(generic_make_request); +#ifdef CONFIG_BLK_DEV_IO_TRACE +static inline struct task_struct *get_dirty_task(struct bio *bio) +{ + /* + * Not all the pages in the bio are dirtied by the + * same task but most likely it will be, since the + * sectors accessed on the device must be adjacent. + */ + if (bio->bi_io_vec && bio->bi_io_vec->bv_page && + bio->bi_io_vec->bv_page->tsk_dirty) + return bio->bi_io_vec->bv_page->tsk_dirty; + else + return current; +} +#else +static inline struct task_struct *get_dirty_task(struct bio *bio) +{ + return current; +} +#endif + +#ifdef CONFIG_BLOCK_PERF_FRAMEWORK +#define BLK_PERF_SIZE (1024 * 15) +#define BLK_PERF_HIST_SIZE (sizeof(u32) * BLK_PERF_SIZE) + +struct blk_perf_stats { + u32 *read_hist; + u32 *write_hist; + u32 *flush_hist; + int buffers_alloced; + ktime_t max_read_time; + ktime_t max_write_time; + ktime_t max_flush_time; + ktime_t min_write_time; + ktime_t min_read_time; + ktime_t min_flush_time; + ktime_t total_write_time; + ktime_t total_read_time; + u64 total_read_size; + u64 total_write_size; + spinlock_t lock; + int is_enabled; +}; + +static struct blk_perf_stats blk_perf; +static struct dentry *blk_perf_debug_dir; + +static int alloc_histogram_buffers(void) +{ + int ret = 0; + + if (!blk_perf.read_hist) + blk_perf.read_hist = kzalloc(BLK_PERF_HIST_SIZE, GFP_KERNEL); + + if (!blk_perf.write_hist) + blk_perf.write_hist = kzalloc(BLK_PERF_HIST_SIZE, GFP_KERNEL); + + if (!blk_perf.flush_hist) + blk_perf.flush_hist = kzalloc(BLK_PERF_HIST_SIZE, GFP_KERNEL); + + if (!blk_perf.read_hist || !blk_perf.write_hist || !blk_perf.flush_hist) + ret = -ENOMEM; + + if (!ret) + blk_perf.buffers_alloced = 1; + return ret; +} + +static void clear_histogram_buffers(void) +{ + if (!blk_perf.buffers_alloced) + return; + memset(blk_perf.read_hist, 0, BLK_PERF_HIST_SIZE); + memset(blk_perf.write_hist, 0, BLK_PERF_HIST_SIZE); + memset(blk_perf.flush_hist, 0, BLK_PERF_HIST_SIZE); +} + +static int enable_perf(void *data, u64 val) +{ + int ret; + + if (!blk_perf.buffers_alloced) + ret = alloc_histogram_buffers(); + + if (ret) + return ret; + + spin_lock(&blk_perf.lock); + blk_perf.is_enabled = val; + spin_unlock(&blk_perf.lock); + return 0; +} + +static int is_perf_enabled(void *data, u64 *val) +{ + spin_lock(&blk_perf.lock); + *val = blk_perf.is_enabled; + spin_unlock(&blk_perf.lock); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(enable_perf_fops, is_perf_enabled, enable_perf, + "%llu\n"); + +static char *blk_debug_buffer; +static u32 blk_debug_data_size; +static DEFINE_MUTEX(blk_perf_debug_buffer_mutex); + +static ssize_t blk_perf_read(struct file *file, char __user *buf, + size_t count, loff_t *file_pos) +{ + ssize_t ret = 0; + + mutex_lock(&blk_perf_debug_buffer_mutex); + ret = simple_read_from_buffer(buf, count, file_pos, blk_debug_buffer, + blk_debug_data_size); + mutex_unlock(&blk_perf_debug_buffer_mutex); + + return ret; +} + +static int blk_debug_buffer_alloc(u32 buffer_size) +{ + int ret = 0; + + mutex_lock(&blk_perf_debug_buffer_mutex); + if (blk_debug_buffer != NULL) { + pr_err("blk_debug_buffer is in use\n"); + ret = -EBUSY; + goto end; + } + blk_debug_buffer = kzalloc(buffer_size, GFP_KERNEL); + if (!blk_debug_buffer) + ret = -ENOMEM; +end: + mutex_unlock(&blk_perf_debug_buffer_mutex); + return ret; +} + +static int blk_perf_close(struct inode *inode, struct file *file) +{ + mutex_lock(&blk_perf_debug_buffer_mutex); + blk_debug_data_size = 0; + kfree(blk_debug_buffer); + blk_debug_buffer = NULL; + mutex_unlock(&blk_perf_debug_buffer_mutex); + return 0; +} + +static u32 fill_basic_perf_info(char *buffer, u32 buffer_size) +{ + u32 size = 0; + + size += scnprintf(buffer + size, buffer_size - size, "\n"); + + spin_lock(&blk_perf.lock); + size += scnprintf(buffer + size, buffer_size - size, + "max_read_time_ms: %llu\n", + ktime_to_ms(blk_perf.max_read_time)); + + size += scnprintf(buffer + size, buffer_size - size, + "min_read_time_ms: %llu\n", + ktime_to_ms(blk_perf.min_read_time)); + + size += scnprintf(buffer + size, buffer_size - size, + "total_read_time_ms: %llu\n", + ktime_to_ms(blk_perf.total_read_time)); + + size += scnprintf(buffer + size, buffer_size - size, + "total_read_size: %llu\n\n", + blk_perf.total_read_size); + + size += scnprintf(buffer + size, buffer_size - size, + "max_write_time_ms: %llu\n", + ktime_to_ms(blk_perf.max_write_time)); + + size += scnprintf(buffer + size, buffer_size - size, + "min_write_time_ms: %llu\n", + ktime_to_ms(blk_perf.min_write_time)); + + size += scnprintf(buffer + size, buffer_size - size, + "total_write_time_ms: %llu\n", + ktime_to_ms(blk_perf.total_write_time)); + + size += scnprintf(buffer + size, buffer_size - size, + "total_write_size: %llu\n\n", + blk_perf.total_write_size); + + size += scnprintf(buffer + size, buffer_size - size, + "max_flush_time_ms: %llu\n", + ktime_to_ms(blk_perf.max_flush_time)); + + size += scnprintf(buffer + size, buffer_size - size, + "min_flush_time_ms: %llu\n\n", + ktime_to_ms(blk_perf.min_flush_time)); + + spin_unlock(&blk_perf.lock); + + return size; +} + +static int basic_perf_open(struct inode *inode, struct file *file) +{ + u32 buffer_size; + int ret; + + buffer_size = BLK_PERF_HIST_SIZE; + ret = blk_debug_buffer_alloc(buffer_size); + if (ret) + return ret; + + mutex_lock(&blk_perf_debug_buffer_mutex); + blk_debug_data_size = fill_basic_perf_info(blk_debug_buffer, + buffer_size); + mutex_unlock(&blk_perf_debug_buffer_mutex); + return 0; +} + + +static const struct file_operations basic_perf_ops = { + .read = blk_perf_read, + .release = blk_perf_close, + .open = basic_perf_open, +}; + +static int hist_open_helper(void *hist_buf) +{ + int ret; + + if (!blk_perf.buffers_alloced) + return -EINVAL; + + ret = blk_debug_buffer_alloc(BLK_PERF_HIST_SIZE); + if (ret) + return ret; + + spin_lock(&blk_perf.lock); + memcpy(blk_debug_buffer, hist_buf, BLK_PERF_HIST_SIZE); + spin_unlock(&blk_perf.lock); + + mutex_lock(&blk_perf_debug_buffer_mutex); + blk_debug_data_size = BLK_PERF_HIST_SIZE; + mutex_unlock(&blk_perf_debug_buffer_mutex); + return 0; +} + +static int write_hist_open(struct inode *inode, struct file *file) +{ + return hist_open_helper(blk_perf.write_hist); +} + +static const struct file_operations write_hist_ops = { + .read = blk_perf_read, + .release = blk_perf_close, + .open = write_hist_open, +}; + + +static int read_hist_open(struct inode *inode, struct file *file) +{ + return hist_open_helper(blk_perf.read_hist); +} + +static const struct file_operations read_hist_ops = { + .read = blk_perf_read, + .release = blk_perf_close, + .open = read_hist_open, +}; + +static int flush_hist_open(struct inode *inode, struct file *file) +{ + return hist_open_helper(blk_perf.flush_hist); +} + +static const struct file_operations flush_hist_ops = { + .read = blk_perf_read, + .release = blk_perf_close, + .open = flush_hist_open, +}; + +static void clear_perf_stats_helper(void) +{ + spin_lock(&blk_perf.lock); + blk_perf.max_write_time = ktime_set(0, 0); + blk_perf.max_read_time = ktime_set(0, 0); + blk_perf.max_flush_time = ktime_set(0, 0); + blk_perf.min_write_time = ktime_set(KTIME_MAX, 0); + blk_perf.min_read_time = ktime_set(KTIME_MAX, 0); + blk_perf.min_flush_time = ktime_set(KTIME_MAX, 0); + blk_perf.total_write_time = ktime_set(0, 0); + blk_perf.total_read_time = ktime_set(0, 0); + blk_perf.total_read_size = 0; + blk_perf.total_write_size = 0; + blk_perf.is_enabled = 0; + clear_histogram_buffers(); + spin_unlock(&blk_perf.lock); +} + +static int clear_perf_stats(void *data, u64 val) +{ + clear_perf_stats_helper(); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(clear_perf_stats_fops, NULL, clear_perf_stats, + "%llu\n"); + +static void blk_debugfs_init(void) +{ + struct dentry *f_ent; + + blk_perf_debug_dir = debugfs_create_dir("block_perf", NULL); + if (IS_ERR(blk_perf_debug_dir)) { + pr_err("Failed to create block_perf debug_fs directory\n"); + return; + } + + f_ent = debugfs_create_file("basic_perf", 0400, blk_perf_debug_dir, + NULL, &basic_perf_ops); + if (IS_ERR(f_ent)) { + pr_err("Failed to create debug_fs basic_perf file\n"); + return; + } + + f_ent = debugfs_create_file("write_hist", 0400, blk_perf_debug_dir, + NULL, &write_hist_ops); + if (IS_ERR(f_ent)) { + pr_err("Failed to create debug_fs write_hist file\n"); + return; + } + + f_ent = debugfs_create_file("read_hist", 0400, blk_perf_debug_dir, + NULL, &read_hist_ops); + if (IS_ERR(f_ent)) { + pr_err("Failed to create debug_fs read_hist file\n"); + return; + } + + f_ent = debugfs_create_file("flush_hist", 0400, blk_perf_debug_dir, + NULL, &flush_hist_ops); + if (IS_ERR(f_ent)) { + pr_err("Failed to create debug_fs flush_hist file\n"); + return; + } + + f_ent = debugfs_create_file("enable_perf", 0600, blk_perf_debug_dir, + NULL, &enable_perf_fops); + if (IS_ERR(f_ent)) { + pr_err("Failed to create debug_fs enable_perf file\n"); + return; + } + + f_ent = debugfs_create_file("clear_perf_stats", 0200, + blk_perf_debug_dir, NULL, + &clear_perf_stats_fops); + if (IS_ERR(f_ent)) { + pr_err("Failed to create debug_fs clear_perf_stats file\n"); + return; + } +} + +static void blk_init_perf(void) +{ + blk_debugfs_init(); + spin_lock_init(&blk_perf.lock); + + clear_perf_stats_helper(); +} + + +static void set_submit_info(struct bio *bio, unsigned int count) +{ + ktime_t submit_time; + + if (unlikely(blk_perf.is_enabled)) { + submit_time = ktime_get(); + bio->submit_time.tv64 = submit_time.tv64; + bio->blk_sector_count = count; + return; + } + + bio->submit_time.tv64 = 0; + bio->blk_sector_count = 0; +} + +void blk_update_perf_read_write_stats(ktime_t bio_process_time, int is_write, + int count) +{ + u32 bio_process_time_ms; + + bio_process_time_ms = ktime_to_ms(bio_process_time); + if (bio_process_time_ms >= BLK_PERF_SIZE) + bio_process_time_ms = BLK_PERF_SIZE - 1; + + if (is_write) { + if (ktime_after(bio_process_time, blk_perf.max_write_time)) + blk_perf.max_write_time = bio_process_time; + + if (ktime_before(bio_process_time, blk_perf.min_write_time)) + blk_perf.min_write_time = bio_process_time; + blk_perf.total_write_time = + ktime_add(blk_perf.total_write_time, bio_process_time); + blk_perf.total_write_size += count; + blk_perf.write_hist[bio_process_time_ms] += count; + + } else { + if (ktime_after(bio_process_time, blk_perf.max_read_time)) + blk_perf.max_read_time = bio_process_time; + + if (ktime_before(bio_process_time, blk_perf.min_read_time)) + blk_perf.min_read_time = bio_process_time; + blk_perf.total_read_time = + ktime_add(blk_perf.total_read_time, bio_process_time); + blk_perf.total_read_size += count; + blk_perf.read_hist[bio_process_time_ms] += count; + } +} +void blk_update_perf_stats(struct bio *bio) +{ + ktime_t bio_process_time; + u32 bio_process_time_ms; + u32 count; + + spin_lock(&blk_perf.lock); + if (likely(!blk_perf.is_enabled)) + goto end; + if (!bio->submit_time.tv64) + goto end; + bio_process_time = ktime_sub(ktime_get(), bio->submit_time); + + count = bio->blk_sector_count; + + if (count) { + int is_write = 0; + + if (bio->bi_rw & WRITE || + unlikely(bio->bi_rw & REQ_WRITE_SAME)) + is_write = 1; + + blk_update_perf_read_write_stats(bio_process_time, is_write, + count); + } else { + + bio_process_time_ms = ktime_to_ms(bio_process_time); + if (bio_process_time_ms >= BLK_PERF_SIZE) + bio_process_time_ms = BLK_PERF_SIZE - 1; + + if (ktime_after(bio_process_time, blk_perf.max_flush_time)) + blk_perf.max_flush_time = bio_process_time; + + if (ktime_before(bio_process_time, blk_perf.min_flush_time)) + blk_perf.min_flush_time = bio_process_time; + + blk_perf.flush_hist[bio_process_time_ms] += 1; + } +end: + spin_unlock(&blk_perf.lock); + +} +#else +static inline void set_submit_info(struct bio *bio, unsigned int count) +{ + (void) bio; + (void) count; +} + +static inline void blk_init_perf(void) +{ +} +#endif /* #ifdef CONFIG_BLOCK_PERF_FRAMEWORK */ + /** * submit_bio - submit a bio to the block device layer for I/O * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) @@ -2117,6 +2602,7 @@ */ blk_qc_t submit_bio(int rw, struct bio *bio) { + unsigned int count = 0; bio->bi_rw |= rw; /* @@ -2124,8 +2610,6 @@ * go through the normal accounting stuff before submission. */ if (bio_has_data(bio)) { - unsigned int count; - if (unlikely(rw & REQ_WRITE_SAME)) count = bdev_logical_block_size(bio->bi_bdev) >> 9; else @@ -2140,8 +2624,11 @@ if (unlikely(block_dump)) { char b[BDEVNAME_SIZE]; + struct task_struct *tsk; + + tsk = get_dirty_task(bio); printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", - current->comm, task_pid_nr(current), + tsk->comm, task_pid_nr(tsk), (rw & WRITE) ? "WRITE" : "READ", (unsigned long long)bio->bi_iter.bi_sector, bdevname(bio->bi_bdev, b), @@ -2149,6 +2636,7 @@ } } + set_submit_info(bio, count); return generic_make_request(bio); } EXPORT_SYMBOL(submit_bio); @@ -2645,6 +3133,15 @@ blk_account_io_completion(req, nr_bytes); total_bytes = 0; + + /* + * Check for this if flagged, Req based dm needs to perform + * post processing, hence dont end bios or request.DM + * layer takes care. + */ + if (bio_flagged(req->bio, BIO_DONTFREE)) + return false; + while (req->bio) { struct bio *bio = req->bio; unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); @@ -2751,7 +3248,7 @@ BUG_ON(blk_queued_rq(req)); if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS) - laptop_io_completion(&req->q->backing_dev_info); + laptop_io_completion(req->q->backing_dev_info); blk_delete_timer(req); @@ -3560,6 +4057,55 @@ blk_requestq_cachep = kmem_cache_create("blkdev_queue", sizeof(struct request_queue), 0, SLAB_PANIC, NULL); - + blk_init_perf(); return 0; } + +/* + * Blk IO latency support. We want this to be as cheap as possible, so doing + * this lockless (and avoiding atomics), a few off by a few errors in this + * code is not harmful, and we don't want to do anything that is + * perf-impactful. + * TODO : If necessary, we can make the histograms per-cpu and aggregate + * them when printing them out. + */ +ssize_t +blk_latency_hist_show(char* name, struct io_latency_state *s, char *buf, + int buf_size) +{ + int i; + int bytes_written = 0; + u_int64_t num_elem, elem; + int pct; + u_int64_t average; + + num_elem = s->latency_elems; + if (num_elem > 0) { + average = div64_u64(s->latency_sum, s->latency_elems); + bytes_written += scnprintf(buf + bytes_written, + buf_size - bytes_written, + "IO svc_time %s Latency Histogram (n = %llu," + " average = %llu):\n", name, num_elem, average); + for (i = 0; + i < ARRAY_SIZE(latency_x_axis_us); + i++) { + elem = s->latency_y_axis[i]; + pct = div64_u64(elem * 100, num_elem); + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "\t< %6lluus%15llu%15d%%\n", + latency_x_axis_us[i], + elem, pct); + } + /* Last element in y-axis table is overflow */ + elem = s->latency_y_axis[i]; + pct = div64_u64(elem * 100, num_elem); + bytes_written += scnprintf(buf + bytes_written, + PAGE_SIZE - bytes_written, + "\t>=%6lluus%15llu%15d%%\n", + latency_x_axis_us[i - 1], elem, pct); + } + + return bytes_written; +} +EXPORT_SYMBOL(blk_latency_hist_show); diff -ruw linux-4.4.115/block/blk-flush.c linux-4.4.115-fbx/block/blk-flush.c --- linux-4.4.115/block/blk-flush.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-flush.c 2019-01-22 16:16:22.715239271 +0100 @@ -62,6 +62,45 @@ * The above peculiarity requires that each FLUSH/FUA request has only one * bio attached to it, which is guaranteed as they aren't allowed to be * merged in the usual way. + * + * Cache Barrier support: + * + * Cache barrier is a requests that instruct the storage devices to apply some + * ordering when writing data from the device's cache to the medium. Write + * requests arriving before a 'cache barrier' request will be written to the + * medium before write requests that will arrive after the 'cache barrier'. + * Since the barrier request is not supported by all block devices, the + * appropriate fallback is flush request. This will make sure application using + * it can relay on correct functionality without consider the specification of + * the device. + * + * If a barrier request is queued, it will follow the same path as a flush + * request. When its time to issue the request, the flush pending list will + * be scanned and if it contains only requests marked with barrier, a barrier + * request will be issued. Otherwise, if at least one flush is pending - flush + * will be issued. + * A barrier request is a flush request marked with the REQ_BARRIER flag. It + * is the LLD responsibility to test this flag if it supports the barrier + * feature and decide whether to issue a flush or a barrier request. + * + * When considering a barrier request, three sequences must be addressed: + * 1. (A)Barrier -> (B)Data, This sequence will be marked with + * WRITE_FLUSH_BARRIER or (REQ_FLUSH | REQ_BARRIER). + * This scenario will be split to a PREFLUSH and DATA and no additional + * execution phase are required. If barrier is not supported, a flush + * will be issued instead (A). + * 2. (A)Data -> (B)Barrier, This sequence will be marked with + * WRITE_POST_FLUSH_BARRIER or (REQ_POST_FLUSH_BARRIER | REQ_BARRIER). + * This request, when barrier is supported, this request will execute DATA + * and than POSTFLUSH. + * If barrier is not supported, but FUA is. The barrier may be replaced + * with DATA+FUA. + * If barrier and FUA are not supported, a flush must be issued instead of + * (B). This is similar to current FUA fallback. + * 3. (A)Barrier -> (B)Data -> (C)Barrier, This sequence will be marked with + * WRITE_ORDERED_FLUSH_BARRIER or (REQ_FLUSH | REQ_POST_FLUSH_BARRIER | + * REQ_BARRIER). This scenario is just a combination of the previous two, + * and no additional logic is required. */ #include @@ -105,8 +144,26 @@ if (fflags & REQ_FLUSH) { if (rq->cmd_flags & REQ_FLUSH) policy |= REQ_FSEQ_PREFLUSH; - if (!(fflags & REQ_FUA) && (rq->cmd_flags & REQ_FUA)) + /* + * Use post flush when: + * 1. If FUA is desired but not supported, + * 2. If post barrier is desired and supported + * 3. If post barrier is desired and not supported and FUA is + * not supported. + */ + if ((!(fflags & REQ_FUA) && (rq->cmd_flags & REQ_FUA)) || + ((fflags & REQ_BARRIER) && (rq->cmd_flags & + REQ_POST_FLUSH_BARRIER)) || + ((!(fflags & REQ_BARRIER) && !(fflags & REQ_FUA) && + (rq->cmd_flags & REQ_POST_FLUSH_BARRIER)))) policy |= REQ_FSEQ_POSTFLUSH; + /* + * If post barrier is desired and not supported but FUA is + * supported append FUA flag. + */ + if ((rq->cmd_flags & REQ_POST_FLUSH_BARRIER) && + !(fflags & REQ_BARRIER) && (fflags & REQ_FUA)) + rq->cmd_flags |= REQ_FUA; } return policy; } @@ -290,9 +347,10 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) { struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; - struct request *first_rq = + struct request *rq, *n, *first_rq = list_first_entry(pending, struct request, flush.list); struct request *flush_rq = fq->flush_rq; + u64 barrier_flag = REQ_BARRIER; /* C1 described at the top of this file */ if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending)) @@ -330,6 +388,12 @@ flush_rq->cmd_type = REQ_TYPE_FS; flush_rq->cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; + /* Issue a barrier only if all pending flushes request it */ + list_for_each_entry_safe(rq, n, pending, flush.list) { + barrier_flag &= rq->cmd_flags; + } + flush_rq->cmd_flags |= barrier_flag; + flush_rq->rq_disk = first_rq->rq_disk; flush_rq->end_io = flush_end_io; @@ -388,6 +452,8 @@ unsigned int policy = blk_flush_policy(fflags, rq); struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx); + WARN_ON((rq->cmd_flags & REQ_POST_FLUSH_BARRIER) && + !blk_rq_sectors(rq)); /* * @policy now records what operations need to be done. Adjust * REQ_FLUSH and FUA for the driver. @@ -447,20 +513,8 @@ blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); } -/** - * blkdev_issue_flush - queue a flush - * @bdev: blockdev to issue flush for - * @gfp_mask: memory allocation flags (for bio_alloc) - * @error_sector: error sector - * - * Description: - * Issue a flush for the block device in question. Caller can supply - * room for storing the error offset in case of a flush error, if they - * wish to. If WAIT flag is not passed then caller may check only what - * request was pushed in some internal queue for later handling. - */ -int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, - sector_t *error_sector) +static int __blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, + sector_t *error_sector, int flush_type) { struct request_queue *q; struct bio *bio; @@ -485,7 +539,7 @@ bio = bio_alloc(gfp_mask, 0); bio->bi_bdev = bdev; - ret = submit_bio_wait(WRITE_FLUSH, bio); + ret = submit_bio_wait(flush_type, bio); /* * The driver must store the error location in ->bi_sector, if @@ -498,6 +552,45 @@ bio_put(bio); return ret; } + +/** + * blkdev_issue_barrier - queue a barrier + * @bdev: blockdev to issue barrier for + * @gfp_mask: memory allocation flags (for bio_alloc) + * @error_sector: error sector + * + * Description: + * If blkdev supports the barrier API, issue barrier, otherwise issue a + * flush Caller can supply room for storing the error offset in case of a + * flush error, if they wish to. If WAIT flag is not passed then caller may + * check only what request was pushed in some internal queue for later + * handling. + */ +int blkdev_issue_barrier(struct block_device *bdev, gfp_t gfp_mask, + sector_t *error_sector) +{ + return __blkdev_issue_flush(bdev, gfp_mask, error_sector, + WRITE_FLUSH_BARRIER); +} +EXPORT_SYMBOL(blkdev_issue_barrier); + +/** + * blkdev_issue_flush - queue a flush + * @bdev: blockdev to issue flush for + * @gfp_mask: memory allocation flags (for bio_alloc) + * @error_sector: error sector + * + * Description: + * Issue a flush for the block device in question. Caller can supply + * room for storing the error offset in case of a flush error, if they + * wish to. If WAIT flag is not passed then caller may check only what + * request was pushed in some internal queue for later handling. + */ +int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, + sector_t *error_sector) +{ + return __blkdev_issue_flush(bdev, gfp_mask, error_sector, WRITE_FLUSH); +} EXPORT_SYMBOL(blkdev_issue_flush); struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, diff -ruw linux-4.4.115/block/blk.h linux-4.4.115-fbx/block/blk.h --- linux-4.4.115/block/blk.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk.h 2019-01-22 16:16:22.723239343 +0100 @@ -112,6 +112,15 @@ void blk_account_io_completion(struct request *req, unsigned int bytes); void blk_account_io_done(struct request *req); +#ifdef CONFIG_BLOCK_PERF_FRAMEWORK +void blk_update_perf_stats(struct bio *bio); +#else +static inline void blk_update_perf_stats(struct bio *bio) +{ + (void) bio; +} +#endif + /* * Internal atomic flags for request handling */ @@ -214,7 +223,6 @@ int attempt_front_merge(struct request_queue *q, struct request *rq); int blk_attempt_req_merge(struct request_queue *q, struct request *rq, struct request *next); -void blk_recalc_rq_segments(struct request *rq); void blk_rq_set_mixed_merge(struct request *rq); bool blk_rq_merge_ok(struct request *rq, struct bio *bio); int blk_try_merge(struct request *rq, struct bio *bio); diff -ruw linux-4.4.115/block/blk-merge.c linux-4.4.115-fbx/block/blk-merge.c --- linux-4.4.115/block/blk-merge.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-merge.c 2019-01-22 16:16:22.719239307 +0100 @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include "blk.h" @@ -488,6 +490,93 @@ } EXPORT_SYMBOL(blk_rq_map_sg); +/* + * map a request to scatterlist without combining PHY CONT + * blocks, return number of sg entries setup. Caller + * must make sure sg can hold rq->nr_phys_segments entries + */ +int blk_rq_map_sg_no_cluster(struct request_queue *q, struct request *rq, + struct scatterlist *sglist) +{ + struct bio_vec bvec, bvprv = { NULL }; + struct req_iterator iter; + struct scatterlist *sg; + int nsegs, cluster = 0; + + nsegs = 0; + + /* + * for each bio in rq + */ + sg = NULL; + rq_for_each_segment(bvec, rq, iter) { + __blk_segment_map_sg(q, &bvec, sglist, &bvprv, &sg, + &nsegs, &cluster); + } /* segments in rq */ + + + if (!sg) + return nsegs; + + if (unlikely(rq->cmd_flags & REQ_COPY_USER) && + (blk_rq_bytes(rq) & q->dma_pad_mask)) { + unsigned int pad_len = + (q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1; + + sg->length += pad_len; + rq->extra_len += pad_len; + } + + if (q->dma_drain_size && q->dma_drain_needed(rq)) { + if (rq->cmd_flags & REQ_WRITE) + memset(q->dma_drain_buffer, 0, q->dma_drain_size); + + sg->page_link &= ~0x02; + sg = sg_next(sg); + sg_set_page(sg, virt_to_page(q->dma_drain_buffer), + q->dma_drain_size, + ((unsigned long)q->dma_drain_buffer) & + (PAGE_SIZE - 1)); + nsegs++; + rq->extra_len += q->dma_drain_size; + } + + if (sg) + sg_mark_end(sg); + + return nsegs; +} +EXPORT_SYMBOL(blk_rq_map_sg_no_cluster); + +/** + * blk_bio_map_sg - map a bio to a scatterlist + * @q: request_queue in question + * @bio: bio being mapped + * @sglist: scatterlist being mapped + * + * Note: + * Caller must make sure sg can hold bio->bi_phys_segments entries + * + * Will return the number of sg entries setup + */ +int blk_bio_map_sg(struct request_queue *q, struct bio *bio, + struct scatterlist *sglist) +{ + struct scatterlist *sg = NULL; + int nsegs; + struct bio *next = bio->bi_next; + + bio->bi_next = NULL; + nsegs = __blk_bios_map_sg(q, bio, sglist, &sg); + bio->bi_next = next; + if (sg) + sg_mark_end(sg); + + WARN_ON(bio->bi_phys_segments && nsegs > bio->bi_phys_segments); + return nsegs; +} +EXPORT_SYMBOL(blk_bio_map_sg); + static inline int ll_new_hw_segment(struct request_queue *q, struct request *req, struct bio *bio) @@ -663,6 +752,12 @@ } } +static bool crypto_not_mergeable(const struct bio *bio, const struct bio *nxt) +{ + return (!pft_allow_merge_bio(bio, nxt) || + !pfk_allow_merge_bio(bio, nxt)); +} + /* * Has to be called with the request spinlock acquired */ @@ -690,6 +785,9 @@ !blk_write_same_mergeable(req->bio, next->bio)) return 0; + if (crypto_not_mergeable(req->bio, next->bio)) + return 0; + /* * If we are allowed to merge, then append bio list * from next to rq and release next. merge_requests_fn @@ -794,6 +892,9 @@ !blk_write_same_mergeable(rq->bio, bio)) return false; + if (crypto_not_mergeable(rq->bio, bio)) + return false; + return true; } diff -ruw linux-4.4.115/block/blk-mq.c linux-4.4.115-fbx/block/blk-mq.c --- linux-4.4.115/block/blk-mq.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-mq.c 2019-10-29 09:26:23.333200144 +0100 @@ -1782,10 +1782,6 @@ INIT_LIST_HEAD(&__ctx->rq_list); __ctx->queue = q; - /* If the cpu isn't online, the cpu is mapped to first hctx */ - if (!cpu_online(i)) - continue; - hctx = q->mq_ops->map_queue(q, i); /* @@ -1819,11 +1815,8 @@ * Map software to hardware queues */ queue_for_each_ctx(q, ctx, i) { - /* If the cpu isn't online, the cpu is mapped to first hctx */ - if (!cpumask_test_cpu(i, online_mask)) - continue; - hctx = q->mq_ops->map_queue(q, i); + if (cpumask_test_cpu(i, online_mask)) cpumask_set_cpu(i, hctx->cpumask); ctx->index_hw = hctx->nr_ctx; hctx->ctxs[hctx->nr_ctx++] = ctx; @@ -1862,16 +1855,21 @@ /* * Initialize batch roundrobin counts + * Set next_cpu for only those hctxs that have an online CPU + * in their cpumask field. For hctxs that belong to few online + * and few offline CPUs, this will always provide one CPU from + * online ones. For hctxs belonging to all offline CPUs, their + * cpumask will be updated in reinit_notify. */ + if (cpumask_first(hctx->cpumask) < nr_cpu_ids) { hctx->next_cpu = cpumask_first(hctx->cpumask); hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; } + } queue_for_each_ctx(q, ctx, i) { - if (!cpumask_test_cpu(i, online_mask)) - continue; - hctx = q->mq_ops->map_queue(q, i); + if (cpumask_test_cpu(i, online_mask)) cpumask_set_cpu(i, hctx->tags->cpumask); } } @@ -2100,38 +2098,13 @@ blk_mq_free_hw_queues(q, set); } -/* Basically redo blk_mq_init_queue with queue frozen */ -static void blk_mq_queue_reinit(struct request_queue *q, - const struct cpumask *online_mask) -{ - WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); - - blk_mq_sysfs_unregister(q); - - blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues, online_mask); - - /* - * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe - * we should change hctx numa_node according to new topology (this - * involves free and re-allocate memory, worthy doing?) - */ - - blk_mq_map_swqueue(q, online_mask); - - blk_mq_sysfs_register(q); -} - static int blk_mq_queue_reinit_notify(struct notifier_block *nb, unsigned long action, void *hcpu) { struct request_queue *q; + struct blk_mq_hw_ctx *hctx; + int i; int cpu = (unsigned long)hcpu; - /* - * New online cpumask which is going to be set in this hotplug event. - * Declare this cpumasks as global as cpu-hotplug operation is invoked - * one-by-one and dynamically allocating this could result in a failure. - */ - static struct cpumask online_new; /* * Before hotadded cpu starts handling requests, new mappings must @@ -2153,44 +2126,31 @@ switch (action & ~CPU_TASKS_FROZEN) { case CPU_DEAD: case CPU_UP_CANCELED: - cpumask_copy(&online_new, cpu_online_mask); + mutex_lock(&all_q_mutex); + list_for_each_entry(q, &all_q_list, all_q_node) { + queue_for_each_hw_ctx(q, hctx, i) { + cpumask_clear_cpu(cpu, hctx->cpumask); + cpumask_clear_cpu(cpu, hctx->tags->cpumask); + } + } + mutex_unlock(&all_q_mutex); break; case CPU_UP_PREPARE: - cpumask_copy(&online_new, cpu_online_mask); - cpumask_set_cpu(cpu, &online_new); + /* Update hctx->cpumask for newly onlined CPUs */ + mutex_lock(&all_q_mutex); + list_for_each_entry(q, &all_q_list, all_q_node) { + queue_for_each_hw_ctx(q, hctx, i) { + cpumask_set_cpu(cpu, hctx->cpumask); + hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; + cpumask_set_cpu(cpu, hctx->tags->cpumask); + } + } + mutex_unlock(&all_q_mutex); break; default: return NOTIFY_OK; } - mutex_lock(&all_q_mutex); - - /* - * We need to freeze and reinit all existing queues. Freezing - * involves synchronous wait for an RCU grace period and doing it - * one by one may take a long time. Start freezing all queues in - * one swoop and then wait for the completions so that freezing can - * take place in parallel. - */ - list_for_each_entry(q, &all_q_list, all_q_node) - blk_mq_freeze_queue_start(q); - list_for_each_entry(q, &all_q_list, all_q_node) { - blk_mq_freeze_queue_wait(q); - - /* - * timeout handler can't touch hw queue during the - * reinitialization - */ - del_timer_sync(&q->timeout); - } - - list_for_each_entry(q, &all_q_list, all_q_node) - blk_mq_queue_reinit(q, &online_new); - - list_for_each_entry(q, &all_q_list, all_q_node) - blk_mq_unfreeze_queue(q); - - mutex_unlock(&all_q_mutex); return NOTIFY_OK; } diff -ruw linux-4.4.115/block/blk-mq-cpumap.c linux-4.4.115-fbx/block/blk-mq-cpumap.c --- linux-4.4.115/block/blk-mq-cpumap.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-mq-cpumap.c 2019-01-22 16:16:22.719239307 +0100 @@ -31,8 +31,8 @@ return cpu; } -int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues, - const struct cpumask *online_mask) +static int blk_mq_update_queue_map(unsigned int *map, + unsigned int nr_queues, const struct cpumask *online_mask) { unsigned int i, nr_cpus, nr_uniq_cpus, queue, first_sibling; cpumask_var_t cpus; @@ -52,18 +52,14 @@ queue = 0; for_each_possible_cpu(i) { - if (!cpumask_test_cpu(i, online_mask)) { - map[i] = 0; - continue; - } - /* * Easy case - we have equal or more hardware queues. Or * there are no thread siblings to take into account. Do * 1:1 if enough, or sequential mapping if less. */ - if (nr_queues >= nr_cpus || nr_cpus == nr_uniq_cpus) { - map[i] = cpu_to_queue_index(nr_cpus, nr_queues, queue); + if (nr_queues >= nr_cpu_ids) { + map[i] = cpu_to_queue_index(nr_cpu_ids, nr_queues, + queue); queue++; continue; } @@ -75,7 +71,7 @@ */ first_sibling = get_first_sibling(i); if (first_sibling == i) { - map[i] = cpu_to_queue_index(nr_uniq_cpus, nr_queues, + map[i] = cpu_to_queue_index(nr_cpu_ids, nr_queues, queue); queue++; } else diff -ruw linux-4.4.115/block/blk-mq.h linux-4.4.115-fbx/block/blk-mq.h --- linux-4.4.115/block/blk-mq.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-mq.h 2019-01-22 16:16:22.719239307 +0100 @@ -48,8 +48,6 @@ * CPU -> queue mappings */ extern unsigned int *blk_mq_make_queue_map(struct blk_mq_tag_set *set); -extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues, - const struct cpumask *online_mask); extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int); /* diff -ruw linux-4.4.115/block/blk-settings.c linux-4.4.115-fbx/block/blk-settings.c --- linux-4.4.115/block/blk-settings.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-settings.c 2019-10-29 09:26:23.337200184 +0100 @@ -823,20 +823,24 @@ /** * blk_queue_flush - configure queue's cache flush capability * @q: the request queue for the device - * @flush: 0, REQ_FLUSH or REQ_FLUSH | REQ_FUA + * @flush: 0, REQ_FLUSH or REQ_FLUSH | REQ_FUA | REQ_BARRIER * * Tell block layer cache flush capability of @q. If it supports * flushing, REQ_FLUSH should be set. If it supports bypassing - * write cache for individual writes, REQ_FUA should be set. + * write cache for individual writes, REQ_FUA should be set. If cache + * barrier is supported set REQ_BARRIER. */ void blk_queue_flush(struct request_queue *q, unsigned int flush) { - WARN_ON_ONCE(flush & ~(REQ_FLUSH | REQ_FUA)); + WARN_ON_ONCE(flush & ~(REQ_FLUSH | REQ_FUA | REQ_BARRIER)); - if (WARN_ON_ONCE(!(flush & REQ_FLUSH) && (flush & REQ_FUA))) + if (WARN_ON_ONCE(!(flush & REQ_FLUSH) && ((flush & REQ_FUA) || + (flush & REQ_BARRIER)))) { flush &= ~REQ_FUA; + flush &= ~REQ_BARRIER; + } - q->flush_flags = flush & (REQ_FLUSH | REQ_FUA); + q->flush_flags = flush & (REQ_FLUSH | REQ_FUA | REQ_BARRIER); } EXPORT_SYMBOL_GPL(blk_queue_flush); diff -ruw linux-4.4.115/block/blk-sysfs.c linux-4.4.115-fbx/block/blk-sysfs.c --- linux-4.4.115/block/blk-sysfs.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/blk-sysfs.c 2019-10-29 09:26:23.337200184 +0100 @@ -75,7 +75,7 @@ static ssize_t queue_ra_show(struct request_queue *q, char *page) { - unsigned long ra_kb = q->backing_dev_info.ra_pages << + unsigned long ra_kb = q->backing_dev_info->ra_pages << (PAGE_CACHE_SHIFT - 10); return queue_var_show(ra_kb, (page)); @@ -90,7 +90,7 @@ if (ret < 0) return ret; - q->backing_dev_info.ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10); + q->backing_dev_info->ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10); return ret; } @@ -578,7 +578,7 @@ struct request_queue *q = container_of(kobj, struct request_queue, kobj); - bdi_exit(&q->backing_dev_info); + bdi_put(q->backing_dev_info); blkcg_exit_queue(q); if (q->elevator) { diff -ruw linux-4.4.115/block/elevator.c linux-4.4.115-fbx/block/elevator.c --- linux-4.4.115/block/elevator.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/elevator.c 2019-01-22 16:16:22.727239379 +0100 @@ -538,7 +538,7 @@ #ifdef CONFIG_PM static void blk_pm_requeue_request(struct request *rq) { - if (rq->q->dev && !(rq->cmd_flags & REQ_PM)) + if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && rq->q->nr_pending) rq->q->nr_pending--; } diff -ruw linux-4.4.115/block/genhd.c linux-4.4.115-fbx/block/genhd.c --- linux-4.4.115/block/genhd.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/genhd.c 2019-01-22 16:16:22.727239379 +0100 @@ -611,7 +611,7 @@ disk_alloc_events(disk); /* Register BDI before referencing it from bdev */ - bdi = &disk->queue->backing_dev_info; + bdi = disk->queue->backing_dev_info; bdi_register_owner(bdi, disk_to_dev(disk)); blk_register_region(disk_devt(disk), disk->minors, NULL, @@ -646,6 +646,8 @@ disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY | DISK_PITER_REVERSE); while ((part = disk_part_iter_next(&piter))) { + bdev_unhash_inode(MKDEV(disk->major, + disk->first_minor + part->partno)); invalidate_partition(disk, part->partno); delete_partition(disk, part->partno); } @@ -656,7 +658,16 @@ disk->flags &= ~GENHD_FL_UP; sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi"); + if (disk->queue) { + /* + * Unregister bdi before releasing device numbers (as they can + * get reused and we'd get clashes in sysfs). + */ + bdi_unregister(disk->queue->backing_dev_info); blk_unregister_queue(disk); + } else { + WARN_ON(1); + } blk_unregister_region(disk_devt(disk), disk->minors); part_stat_set_all(&disk->part0, 0); @@ -1117,6 +1128,22 @@ blk_put_queue(disk->queue); kfree(disk); } + +static int disk_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + struct gendisk *disk = dev_to_disk(dev); + struct disk_part_iter piter; + struct hd_struct *part; + int cnt = 0; + + disk_part_iter_init(&piter, disk, 0); + while((part = disk_part_iter_next(&piter))) + cnt++; + disk_part_iter_exit(&piter); + add_uevent_var(env, "NPARTS=%u", cnt); + return 0; +} + struct class block_class = { .name = "block", }; @@ -1136,6 +1163,7 @@ .groups = disk_attr_groups, .release = disk_release, .devnode = block_devnode, + .uevent = disk_uevent, }; #ifdef CONFIG_PROC_FS diff -ruw linux-4.4.115/block/Kconfig linux-4.4.115-fbx/block/Kconfig --- linux-4.4.115/block/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/Kconfig 2019-01-22 16:16:22.707239198 +0100 @@ -111,6 +111,13 @@ See Documentation/block/cmdline-partition.txt for more information. +config BLOCK_PERF_FRAMEWORK + bool "Enable Block device performance measurement framework" + default n + ---help--- + Enabling this option allows you to measure the performance at the + block layer. + menu "Partition Types" source "block/partitions/Kconfig" diff -ruw linux-4.4.115/block/Kconfig.iosched linux-4.4.115-fbx/block/Kconfig.iosched --- linux-4.4.115/block/Kconfig.iosched 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/Kconfig.iosched 2019-01-22 16:16:22.711239234 +0100 @@ -12,6 +12,17 @@ that do their own scheduling and require only minimal assistance from the kernel. +config IOSCHED_TEST + tristate "Test I/O scheduler" + depends on DEBUG_FS + default m + ---help--- + The test I/O scheduler is a duplicate of the noop scheduler with + addition of test utlity. + It allows testing a block device by dispatching specific requests + according to the test case and declare PASS/FAIL according to the + requests completion error code. + config IOSCHED_DEADLINE tristate "Deadline I/O scheduler" default y diff -ruw linux-4.4.115/block/Makefile linux-4.4.115-fbx/block/Makefile --- linux-4.4.115/block/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/Makefile 2019-01-22 16:16:22.711239234 +0100 @@ -18,6 +18,7 @@ obj-$(CONFIG_IOSCHED_NOOP) += noop-iosched.o obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o +obj-$(CONFIG_IOSCHED_TEST) += test-iosched.o obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o diff -ruw linux-4.4.115/block/partition-generic.c linux-4.4.115-fbx/block/partition-generic.c --- linux-4.4.115/block/partition-generic.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/block/partition-generic.c 2019-10-29 09:26:23.341200223 +0100 @@ -216,10 +216,21 @@ kfree(p); } +static int part_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + struct hd_struct *part = dev_to_part(dev); + + add_uevent_var(env, "PARTN=%u", part->partno); + if (part->info && part->info->volname[0]) + add_uevent_var(env, "PARTNAME=%s", part->info->volname); + return 0; +} + struct device_type part_type = { .name = "partition", .groups = part_attr_groups, .release = part_release, + .uevent = part_uevent, }; static void delete_partition_rcu_cb(struct rcu_head *head) diff -ruw linux-4.4.115/certs/Makefile linux-4.4.115-fbx/certs/Makefile --- linux-4.4.115/certs/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/certs/Makefile 2019-10-29 09:26:23.341200223 +0100 @@ -17,6 +17,10 @@ quiet_cmd_extract_certs = EXTRACT_CERTS $(patsubst "%",%,$(2)) cmd_extract_certs = scripts/extract-cert $(2) $@ || ( rm $@; exit 1) +ifeq ($(CONFIG_SYSTEM_TRUSTED_KEYS),"verity.x509.pem") +SYSTEM_TRUSTED_KEYS_SRCPREFIX := $(srctree)/certs/ +endif + targets += x509_certificate_list $(obj)/x509_certificate_list: scripts/extract-cert $(SYSTEM_TRUSTED_KEYS_SRCPREFIX)$(SYSTEM_TRUSTED_KEYS_FILENAME) FORCE $(call if_changed,extract_certs,$(SYSTEM_TRUSTED_KEYS_SRCPREFIX)$(CONFIG_SYSTEM_TRUSTED_KEYS)) diff -ruw linux-4.4.115/crypto/api.c linux-4.4.115-fbx/crypto/api.c --- linux-4.4.115/crypto/api.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/crypto/api.c 2019-01-22 16:16:22.739239488 +0100 @@ -24,6 +24,7 @@ #include #include #include +#include #include "internal.h" LIST_HEAD(crypto_alg_list); @@ -611,5 +612,17 @@ } EXPORT_SYMBOL_GPL(crypto_has_alg); +void crypto_req_done(struct crypto_async_request *req, int err) +{ + struct crypto_wait *wait = req->data; + + if (err == -EINPROGRESS) + return; + + wait->err = err; + complete(&wait->completion); +} +EXPORT_SYMBOL_GPL(crypto_req_done); + MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff -ruw linux-4.4.115/crypto/blkcipher.c linux-4.4.115-fbx/crypto/blkcipher.c --- linux-4.4.115/crypto/blkcipher.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/crypto/blkcipher.c 2019-10-29 09:26:23.345200262 +0100 @@ -373,6 +373,27 @@ } EXPORT_SYMBOL_GPL(blkcipher_aead_walk_virt_block); +/* + * This function allows ablkcipher algorithms to use the blkcipher_walk API to + * walk over their data. The specified crypto_ablkcipher tfm is used to + * initialize the struct blkcipher_walk, and the crypto_blkcipher specified in + * desc->tfm is never used so it can be left NULL. (Yes, this design is ugly, + * but it parallels blkcipher_aead_walk_virt_block() above. In the 4.10 kernel + * this is starting to be cleaned up...) + */ +int blkcipher_ablkcipher_walk_virt(struct blkcipher_desc *desc, + struct blkcipher_walk *walk, + struct crypto_ablkcipher *tfm) +{ + walk->flags &= ~BLKCIPHER_WALK_PHYS; + walk->walk_blocksize = crypto_ablkcipher_blocksize(tfm); + walk->cipher_blocksize = walk->walk_blocksize; + walk->ivsize = crypto_ablkcipher_ivsize(tfm); + walk->alignmask = crypto_ablkcipher_alignmask(tfm); + return blkcipher_walk_first(desc, walk); +} +EXPORT_SYMBOL_GPL(blkcipher_ablkcipher_walk_virt); + static int setkey_unaligned(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { diff -ruw linux-4.4.115/crypto/gf128mul.c linux-4.4.115-fbx/crypto/gf128mul.c --- linux-4.4.115/crypto/gf128mul.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/crypto/gf128mul.c 2019-01-22 16:16:22.747239560 +0100 @@ -44,7 +44,7 @@ --------------------------------------------------------------------------- Issue 31/01/2006 - This file provides fast multiplication in GF(128) as required by several + This file provides fast multiplication in GF(2^128) as required by several cryptographic authentication modes */ @@ -88,37 +88,52 @@ q(0xf8), q(0xf9), q(0xfa), q(0xfb), q(0xfc), q(0xfd), q(0xfe), q(0xff) \ } -/* Given the value i in 0..255 as the byte overflow when a field element - in GHASH is multiplied by x^8, this function will return the values that - are generated in the lo 16-bit word of the field value by applying the - modular polynomial. The values lo_byte and hi_byte are returned via the - macro xp_fun(lo_byte, hi_byte) so that the values can be assembled into - memory as required by a suitable definition of this macro operating on - the table above +/* + * Given a value i in 0..255 as the byte overflow when a field element + * in GF(2^128) is multiplied by x^8, the following macro returns the + * 16-bit value that must be XOR-ed into the low-degree end of the + * product to reduce it modulo the irreducible polynomial x^128 + x^7 + + * x^2 + x + 1. + * + * There are two versions of the macro, and hence two tables: one for + * the "be" convention where the highest-order bit is the coefficient of + * the highest-degree polynomial term, and one for the "le" convention + * where the highest-order bit is the coefficient of the lowest-degree + * polynomial term. In both cases the values are stored in CPU byte + * endianness such that the coefficients are ordered consistently across + * bytes, i.e. in the "be" table bits 15..0 of the stored value + * correspond to the coefficients of x^15..x^0, and in the "le" table + * bits 15..0 correspond to the coefficients of x^0..x^15. + * + * Therefore, provided that the appropriate byte endianness conversions + * are done by the multiplication functions (and these must be in place + * anyway to support both little endian and big endian CPUs), the "be" + * table can be used for multiplications of both "bbe" and "ble" + * elements, and the "le" table can be used for multiplications of both + * "lle" and "lbe" elements. */ -#define xx(p, q) 0x##p##q - -#define xda_bbe(i) ( \ - (i & 0x80 ? xx(43, 80) : 0) ^ (i & 0x40 ? xx(21, c0) : 0) ^ \ - (i & 0x20 ? xx(10, e0) : 0) ^ (i & 0x10 ? xx(08, 70) : 0) ^ \ - (i & 0x08 ? xx(04, 38) : 0) ^ (i & 0x04 ? xx(02, 1c) : 0) ^ \ - (i & 0x02 ? xx(01, 0e) : 0) ^ (i & 0x01 ? xx(00, 87) : 0) \ +#define xda_be(i) ( \ + (i & 0x80 ? 0x4380 : 0) ^ (i & 0x40 ? 0x21c0 : 0) ^ \ + (i & 0x20 ? 0x10e0 : 0) ^ (i & 0x10 ? 0x0870 : 0) ^ \ + (i & 0x08 ? 0x0438 : 0) ^ (i & 0x04 ? 0x021c : 0) ^ \ + (i & 0x02 ? 0x010e : 0) ^ (i & 0x01 ? 0x0087 : 0) \ ) -#define xda_lle(i) ( \ - (i & 0x80 ? xx(e1, 00) : 0) ^ (i & 0x40 ? xx(70, 80) : 0) ^ \ - (i & 0x20 ? xx(38, 40) : 0) ^ (i & 0x10 ? xx(1c, 20) : 0) ^ \ - (i & 0x08 ? xx(0e, 10) : 0) ^ (i & 0x04 ? xx(07, 08) : 0) ^ \ - (i & 0x02 ? xx(03, 84) : 0) ^ (i & 0x01 ? xx(01, c2) : 0) \ +#define xda_le(i) ( \ + (i & 0x80 ? 0xe100 : 0) ^ (i & 0x40 ? 0x7080 : 0) ^ \ + (i & 0x20 ? 0x3840 : 0) ^ (i & 0x10 ? 0x1c20 : 0) ^ \ + (i & 0x08 ? 0x0e10 : 0) ^ (i & 0x04 ? 0x0708 : 0) ^ \ + (i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \ ) -static const u16 gf128mul_table_lle[256] = gf128mul_dat(xda_lle); -static const u16 gf128mul_table_bbe[256] = gf128mul_dat(xda_bbe); +static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le); +static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be); -/* These functions multiply a field element by x, by x^4 and by x^8 - * in the polynomial field representation. It uses 32-bit word operations - * to gain speed but compensates for machine endianess and hence works +/* + * The following functions multiply a field element by x or by x^8 in + * the polynomial field representation. They use 64-bit word operations + * to gain speed but compensate for machine endianness and hence work * correctly on both styles of machine. */ @@ -126,7 +141,7 @@ { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_lle[(b << 7) & 0xff]; + u64 _tt = gf128mul_table_le[(b << 7) & 0xff]; r->b = cpu_to_be64((b >> 1) | (a << 63)); r->a = cpu_to_be64((a >> 1) ^ (_tt << 48)); @@ -136,7 +151,7 @@ { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_bbe[a >> 63]; + u64 _tt = gf128mul_table_be[a >> 63]; r->a = cpu_to_be64((a << 1) | (b >> 63)); r->b = cpu_to_be64((b << 1) ^ _tt); @@ -146,7 +161,7 @@ { u64 a = le64_to_cpu(x->a); u64 b = le64_to_cpu(x->b); - u64 _tt = gf128mul_table_bbe[b >> 63]; + u64 _tt = gf128mul_table_be[b >> 63]; r->a = cpu_to_le64((a << 1) ^ _tt); r->b = cpu_to_le64((b << 1) | (a >> 63)); @@ -157,7 +172,7 @@ { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_lle[b & 0xff]; + u64 _tt = gf128mul_table_le[b & 0xff]; x->b = cpu_to_be64((b >> 8) | (a << 56)); x->a = cpu_to_be64((a >> 8) ^ (_tt << 48)); @@ -167,12 +182,22 @@ { u64 a = be64_to_cpu(x->a); u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_bbe[a >> 56]; + u64 _tt = gf128mul_table_be[a >> 56]; x->a = cpu_to_be64((a << 8) | (b >> 56)); x->b = cpu_to_be64((b << 8) ^ _tt); } +static void gf128mul_x8_ble(be128 *x) +{ + u64 a = le64_to_cpu(x->b); + u64 b = le64_to_cpu(x->a); + u64 _tt = gf128mul_table_be[a >> 56]; + + x->b = cpu_to_le64((a << 8) | (b >> 56)); + x->a = cpu_to_le64((b << 8) ^ _tt); +} + void gf128mul_lle(be128 *r, const be128 *b) { be128 p[8]; @@ -249,9 +274,48 @@ } EXPORT_SYMBOL(gf128mul_bbe); +void gf128mul_ble(be128 *r, const be128 *b) +{ + be128 p[8]; + int i; + + p[0] = *r; + for (i = 0; i < 7; ++i) + gf128mul_x_ble((be128 *)&p[i + 1], (be128 *)&p[i]); + + memset(r, 0, sizeof(*r)); + for (i = 0;;) { + u8 ch = ((u8 *)b)[15 - i]; + + if (ch & 0x80) + be128_xor(r, r, &p[7]); + if (ch & 0x40) + be128_xor(r, r, &p[6]); + if (ch & 0x20) + be128_xor(r, r, &p[5]); + if (ch & 0x10) + be128_xor(r, r, &p[4]); + if (ch & 0x08) + be128_xor(r, r, &p[3]); + if (ch & 0x04) + be128_xor(r, r, &p[2]); + if (ch & 0x02) + be128_xor(r, r, &p[1]); + if (ch & 0x01) + be128_xor(r, r, &p[0]); + + if (++i >= 16) + break; + + gf128mul_x8_ble(r); + } +} +EXPORT_SYMBOL(gf128mul_ble); + + /* This version uses 64k bytes of table space. A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(128). If we consider a GF(128) value in + value in GF(2^128). If we consider a GF(2^128) value in the buffer's lowest byte, we can construct a table of the 256 16 byte values that result from the 256 values of this byte. This requires 4096 bytes. But we also @@ -352,8 +416,8 @@ int i; for (i = 0; i < 16; i++) - kfree(t->t[i]); - kfree(t); + kzfree(t->t[i]); + kzfree(t); } EXPORT_SYMBOL(gf128mul_free_64k); @@ -385,7 +449,7 @@ /* This version uses 4k bytes of table space. A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(128). If we consider a GF(128) value in a + value in GF(2^128). If we consider a GF(2^128) value in a single byte, we can construct a table of the 256 16 byte values that result from the 256 values of this byte. This requires 4096 bytes. If we take the highest byte in @@ -443,6 +507,28 @@ } EXPORT_SYMBOL(gf128mul_init_4k_bbe); +struct gf128mul_4k *gf128mul_init_4k_ble(const be128 *g) +{ + struct gf128mul_4k *t; + int j, k; + + t = kzalloc(sizeof(*t), GFP_KERNEL); + if (!t) + goto out; + + t->t[1] = *g; + for (j = 1; j <= 64; j <<= 1) + gf128mul_x_ble(&t->t[j + j], &t->t[j]); + + for (j = 2; j < 256; j += j) + for (k = 1; k < j; ++k) + be128_xor(&t->t[j + k], &t->t[j], &t->t[k]); + +out: + return t; +} +EXPORT_SYMBOL(gf128mul_init_4k_ble); + void gf128mul_4k_lle(be128 *a, struct gf128mul_4k *t) { u8 *ap = (u8 *)a; @@ -473,5 +559,20 @@ } EXPORT_SYMBOL(gf128mul_4k_bbe); +void gf128mul_4k_ble(be128 *a, struct gf128mul_4k *t) +{ + u8 *ap = (u8 *)a; + be128 r[1]; + int i = 15; + + *r = t->t[ap[15]]; + while (i--) { + gf128mul_x8_ble(r); + be128_xor(r, r, &t->t[ap[i]]); + } + *a = *r; +} +EXPORT_SYMBOL(gf128mul_4k_ble); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Functions for multiplying elements of GF(2^128)"); diff -ruw linux-4.4.115/crypto/Kconfig linux-4.4.115-fbx/crypto/Kconfig --- linux-4.4.115/crypto/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/crypto/Kconfig 2019-10-29 09:26:23.345200262 +0100 @@ -289,6 +289,24 @@ CBC: Cipher Block Chaining mode This block cipher algorithm is required for IPSec. +config CRYPTO_HEH + tristate "HEH support" + select CRYPTO_CMAC + select CRYPTO_ECB + select CRYPTO_GF128MUL + select CRYPTO_MANAGER + select CRYPTO_POLY_HASH_ARM64_CE if ARM64 && KERNEL_MODE_NEON + help + HEH: Hash-Encrypt-Hash mode + HEH is a proposed block cipher mode of operation which extends the + strong pseudo-random permutation (SPRP) property of block ciphers to + arbitrary-length input strings. This provides a stronger notion of + security than existing block cipher modes of operation (e.g. CBC, CTR, + XTS), though it is usually less performant. Applications include disk + encryption and encryption of file names and contents. Currently, this + implementation only provides a symmetric cipher interface, so it can't + yet be used as an AEAD. + config CRYPTO_CTR tristate "CTR support" select CRYPTO_BLKCIPHER diff -ruw linux-4.4.115/crypto/Makefile linux-4.4.115-fbx/crypto/Makefile --- linux-4.4.115/crypto/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/crypto/Makefile 2019-10-29 09:26:23.345200262 +0100 @@ -67,6 +67,7 @@ obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o obj-$(CONFIG_CRYPTO_ECB) += ecb.o obj-$(CONFIG_CRYPTO_CBC) += cbc.o +obj-$(CONFIG_CRYPTO_HEH) += heh.o obj-$(CONFIG_CRYPTO_PCBC) += pcbc.o obj-$(CONFIG_CRYPTO_CTS) += cts.o obj-$(CONFIG_CRYPTO_LRW) += lrw.o diff -ruw linux-4.4.115/crypto/shash.c linux-4.4.115-fbx/crypto/shash.c --- linux-4.4.115/crypto/shash.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/crypto/shash.c 2019-10-29 09:26:23.349200301 +0100 @@ -686,6 +686,14 @@ } EXPORT_SYMBOL_GPL(shash_free_instance); +int crypto_grab_shash(struct crypto_shash_spawn *spawn, + const char *name, u32 type, u32 mask) +{ + spawn->base.frontend = &crypto_shash_type; + return crypto_grab_spawn(&spawn->base, name, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_grab_shash); + int crypto_init_shash_spawn(struct crypto_shash_spawn *spawn, struct shash_alg *alg, struct crypto_instance *inst) diff -ruw linux-4.4.115/crypto/testmgr.c linux-4.4.115-fbx/crypto/testmgr.c --- linux-4.4.115/crypto/testmgr.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/crypto/testmgr.c 2019-10-29 09:26:23.353200340 +0100 @@ -3215,6 +3215,21 @@ } } }, { + .alg = "heh(aes)", + .test = alg_test_skcipher, + .suite = { + .cipher = { + .enc = { + .vecs = aes_heh_enc_tv_template, + .count = AES_HEH_ENC_TEST_VECTORS + }, + .dec = { + .vecs = aes_heh_dec_tv_template, + .count = AES_HEH_DEC_TEST_VECTORS + } + } + } + }, { .alg = "hmac(crc32)", .test = alg_test_hash, .suite = { diff -ruw linux-4.4.115/drivers/amba/bus.c linux-4.4.115-fbx/drivers/amba/bus.c --- linux-4.4.115/drivers/amba/bus.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/amba/bus.c 2019-10-29 09:26:23.381200614 +0100 @@ -82,7 +82,8 @@ struct amba_device *dev = to_amba_device(_dev); char *driver_override, *old = dev->driver_override, *cp; - if (count > PATH_MAX) + /* We need to keep extra room for a newline */ + if (count >= (PAGE_SIZE - 1)) return -EINVAL; driver_override = kstrndup(buf, count, GFP_KERNEL); diff -ruw linux-4.4.115/drivers/android/Kconfig linux-4.4.115-fbx/drivers/android/Kconfig --- linux-4.4.115/drivers/android/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/android/Kconfig 2019-10-29 09:26:23.381200614 +0100 @@ -19,6 +19,18 @@ Android process, using Binder to identify, invoke and pass arguments between said processes. +config ANDROID_BINDER_DEVICES + string "Android Binder devices" + depends on ANDROID_BINDER_IPC + default "binder,hwbinder,vndbinder" + ---help--- + Default value for the binder.devices parameter. + + The binder.devices parameter is a comma-separated list of strings + that specifies the names of the binder device nodes that will be + created. Each binder device has its own context manager, and is + therefore logically separated from the other devices. + config ANDROID_BINDER_IPC_32BIT bool depends on !64BIT && ANDROID_BINDER_IPC @@ -32,6 +44,16 @@ Note that enabling this will break newer Android user-space. +config ANDROID_BINDER_IPC_SELFTEST + bool "Android Binder IPC Driver Selftest" + depends on ANDROID_BINDER_IPC + ---help--- + This feature allows binder selftest to run. + + Binder selftest checks the allocation and free of binder buffers + exhaustively with combinations of various buffer sizes and + alignments. + endif # if ANDROID endmenu diff -ruw linux-4.4.115/drivers/android/Makefile linux-4.4.115-fbx/drivers/android/Makefile --- linux-4.4.115/drivers/android/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/android/Makefile 2019-01-22 16:16:22.823240249 +0100 @@ -1,3 +1,4 @@ ccflags-y += -I$(src) # needed for trace events -obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o +obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o binder_alloc.o +obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o diff -ruw linux-4.4.115/drivers/base/core.c linux-4.4.115-fbx/drivers/base/core.c --- linux-4.4.115/drivers/base/core.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/core.c 2019-10-29 09:26:23.409200888 +0100 @@ -72,6 +72,11 @@ return restart_syscall(); } +void lock_device_hotplug_assert(void) +{ + lockdep_assert_held(&device_hotplug_lock); +} + #ifdef CONFIG_BLOCK static inline int device_is_not_partition(struct device *dev) { diff -ruw linux-4.4.115/drivers/base/cpu.c linux-4.4.115-fbx/drivers/base/cpu.c --- linux-4.4.115/drivers/base/cpu.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/cpu.c 2019-10-29 09:26:23.409200888 +0100 @@ -180,10 +180,177 @@ }; #endif +#ifdef CONFIG_HOTPLUG_CPU + +static ssize_t isolate_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + ssize_t rc; + int cpuid = cpu->dev.id; + unsigned int isolated = cpu_isolated(cpuid); + + rc = snprintf(buf, PAGE_SIZE-2, "%d\n", isolated); + + return rc; +} + +static DEVICE_ATTR_RO(isolate); + +static struct attribute *cpu_isolated_attrs[] = { + &dev_attr_isolate.attr, + NULL +}; + +static struct attribute_group cpu_isolated_attr_group = { + .attrs = cpu_isolated_attrs, +}; + +#endif + +#ifdef CONFIG_SCHED_HMP + +static ssize_t show_sched_static_cpu_pwr_cost(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + ssize_t rc; + int cpuid = cpu->dev.id; + unsigned int pwr_cost; + + pwr_cost = sched_get_static_cpu_pwr_cost(cpuid); + + rc = snprintf(buf, PAGE_SIZE-2, "%d\n", pwr_cost); + + return rc; +} + +static ssize_t __ref store_sched_static_cpu_pwr_cost(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + int err; + int cpuid = cpu->dev.id; + unsigned int pwr_cost; + + err = kstrtouint(strstrip((char *)buf), 0, &pwr_cost); + if (err) + return err; + + err = sched_set_static_cpu_pwr_cost(cpuid, pwr_cost); + + if (err >= 0) + err = count; + + return err; +} + +static ssize_t show_sched_static_cluster_pwr_cost(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + ssize_t rc; + int cpuid = cpu->dev.id; + unsigned int pwr_cost; + + pwr_cost = sched_get_static_cluster_pwr_cost(cpuid); + + rc = snprintf(buf, PAGE_SIZE-2, "%d\n", pwr_cost); + + return rc; +} + +static ssize_t __ref store_sched_static_cluster_pwr_cost(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + int err; + int cpuid = cpu->dev.id; + unsigned int pwr_cost; + + err = kstrtouint(strstrip((char *)buf), 0, &pwr_cost); + if (err) + return err; + + err = sched_set_static_cluster_pwr_cost(cpuid, pwr_cost); + + if (err >= 0) + err = count; + + return err; +} + +static ssize_t show_sched_cluser_wake_idle(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + ssize_t rc; + int cpuid = cpu->dev.id; + unsigned int wake_up_idle; + + wake_up_idle = sched_get_cluster_wake_idle(cpuid); + + rc = scnprintf(buf, PAGE_SIZE-2, "%d\n", wake_up_idle); + + return rc; +} + +static ssize_t __ref store_sched_cluster_wake_idle(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + int err; + int cpuid = cpu->dev.id; + unsigned int wake_up_idle; + + err = kstrtouint(strstrip((char *)buf), 0, &wake_up_idle); + if (err) + return err; + + err = sched_set_cluster_wake_idle(cpuid, wake_up_idle); + + if (err >= 0) + err = count; + + return err; +} + +static DEVICE_ATTR(sched_static_cpu_pwr_cost, 0644, + show_sched_static_cpu_pwr_cost, + store_sched_static_cpu_pwr_cost); +static DEVICE_ATTR(sched_static_cluster_pwr_cost, 0644, + show_sched_static_cluster_pwr_cost, + store_sched_static_cluster_pwr_cost); +static DEVICE_ATTR(sched_cluster_wake_up_idle, 0644, + show_sched_cluser_wake_idle, + store_sched_cluster_wake_idle); + +static struct attribute *hmp_sched_cpu_attrs[] = { + &dev_attr_sched_static_cpu_pwr_cost.attr, + &dev_attr_sched_static_cluster_pwr_cost.attr, + &dev_attr_sched_cluster_wake_up_idle.attr, + NULL +}; + +static struct attribute_group sched_hmp_cpu_attr_group = { + .attrs = hmp_sched_cpu_attrs, +}; + +#endif /* CONFIG_SCHED_HMP */ + static const struct attribute_group *common_cpu_attr_groups[] = { #ifdef CONFIG_KEXEC &crash_note_cpu_attr_group, #endif +#ifdef CONFIG_SCHED_HMP + &sched_hmp_cpu_attr_group, +#endif +#ifdef CONFIG_HOTPLUG_CPU + &cpu_isolated_attr_group, +#endif NULL }; @@ -191,6 +358,12 @@ #ifdef CONFIG_KEXEC &crash_note_cpu_attr_group, #endif +#ifdef CONFIG_SCHED_HMP + &sched_hmp_cpu_attr_group, +#endif +#ifdef CONFIG_HOTPLUG_CPU + &cpu_isolated_attr_group, +#endif NULL }; @@ -220,6 +393,7 @@ _CPU_ATTR(online, &cpu_online_mask), _CPU_ATTR(possible, &cpu_possible_mask), _CPU_ATTR(present, &cpu_present_mask), + _CPU_ATTR(core_ctl_isolated, &cpu_isolated_mask), }; /* @@ -454,6 +628,7 @@ &cpu_attrs[0].attr.attr, &cpu_attrs[1].attr.attr, &cpu_attrs[2].attr.attr, + &cpu_attrs[3].attr.attr, &dev_attr_kernel_max.attr, &dev_attr_offline.attr, &dev_attr_isolated.attr, diff -ruw linux-4.4.115/drivers/base/dd.c linux-4.4.115-fbx/drivers/base/dd.c --- linux-4.4.115/drivers/base/dd.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/dd.c 2019-10-29 09:26:23.409200888 +0100 @@ -171,26 +171,49 @@ queue_work(deferred_wq, &deferred_probe_work); } +static void enable_trigger_defer_cycle(void) +{ + driver_deferred_probe_enable = true; + driver_deferred_probe_trigger(); + /* + * Sort as many dependencies as possible before the next initcall + * level + */ + flush_workqueue(deferred_wq); +} + /** * deferred_probe_initcall() - Enable probing of deferred devices * * We don't want to get in the way when the bulk of drivers are getting probed. * Instead, this initcall makes sure that deferred probing is delayed until - * late_initcall time. + * all the registered initcall functions at a particular level are completed. + * This function is invoked at every *_initcall_sync level. */ static int deferred_probe_initcall(void) { + if (!deferred_wq) { deferred_wq = create_singlethread_workqueue("deferwq"); if (WARN_ON(!deferred_wq)) return -ENOMEM; + } - driver_deferred_probe_enable = true; - driver_deferred_probe_trigger(); - /* Sort as many dependencies as possible before exiting initcalls */ - flush_workqueue(deferred_wq); + enable_trigger_defer_cycle(); + driver_deferred_probe_enable = false; + return 0; +} +arch_initcall_sync(deferred_probe_initcall); +subsys_initcall_sync(deferred_probe_initcall); +fs_initcall_sync(deferred_probe_initcall); +device_initcall_sync(deferred_probe_initcall); + +static int deferred_probe_enable_fn(void) +{ + /* Enable deferred probing for all time */ + enable_trigger_defer_cycle(); return 0; } -late_initcall(deferred_probe_initcall); +late_initcall(deferred_probe_enable_fn); static void driver_bound(struct device *dev) { @@ -205,6 +228,8 @@ klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices); + device_pm_check_callbacks(dev); + /* * Make sure the device is no longer in one of the deferred lists and * kick off retrying all pending devices @@ -697,6 +722,7 @@ dev->pm_domain->dismiss(dev); klist_remove(&dev->p->knode_driver); + device_pm_check_callbacks(dev); if (dev->bus) blocking_notifier_call_chain(&dev->bus->p->bus_notifier, BUS_NOTIFY_UNBOUND_DRIVER, diff -ruw linux-4.4.115/drivers/base/dma-mapping.c linux-4.4.115-fbx/drivers/base/dma-mapping.c --- linux-4.4.115/drivers/base/dma-mapping.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/dma-mapping.c 2019-01-22 16:16:22.875240720 +0100 @@ -309,7 +309,12 @@ void *ptr; unsigned long pfn; - pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL); + pages = kmalloc(sizeof(struct page *) << get_order(size), + GFP_KERNEL | __GFP_NOWARN); + + if (!pages) + pages = vmalloc(sizeof(struct page *) << get_order(size)); + if (!pages) return NULL; @@ -318,20 +323,24 @@ ptr = dma_common_pages_remap(pages, size, vm_flags, prot, caller); - kfree(pages); + kvfree(pages); return ptr; } /* + area->pages = pages; + * unmaps a range previously mapped by dma_common_*_remap */ -void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags) +void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags, + bool no_warn) { struct vm_struct *area = find_vm_area(cpu_addr); if (!area || (area->flags & vm_flags) != vm_flags) { - WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); + WARN(!no_warn, "trying to free invalid coherent area: %p\n", + cpu_addr); return; } diff -ruw linux-4.4.115/drivers/base/firmware_class.c linux-4.4.115-fbx/drivers/base/firmware_class.c --- linux-4.4.115/drivers/base/firmware_class.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/firmware_class.c 2019-10-29 09:26:23.413200927 +0100 @@ -29,6 +29,7 @@ #include #include #include +#include #include @@ -111,6 +112,7 @@ #define FW_OPT_FALLBACK 0 #endif #define FW_OPT_NO_WARN (1U << 3) +#define FW_OPT_NOCACHE (1U << 4) struct firmware_cache { /* firmware_buf instance will be added into the below list */ @@ -142,6 +144,11 @@ unsigned long status; void *data; size_t size; + phys_addr_t dest_addr; + size_t dest_size; + void * (*map_fw_mem)(phys_addr_t phys, size_t size, void *data); + void (*unmap_fw_mem)(void *virt, size_t size, void *data); + void *map_data; #ifdef CONFIG_FW_LOADER_USER_HELPER bool is_paged_buf; bool need_uevent; @@ -163,6 +170,22 @@ const char *name; }; +struct fw_desc { + struct work_struct work; + const struct firmware **firmware_p; + const char *name; + struct device *device; + unsigned int opt_flags; + phys_addr_t dest_addr; + size_t dest_size; + void * (*map_fw_mem)(phys_addr_t phys, size_t size, void *data); + void (*unmap_fw_mem)(void *virt, size_t size, void *data); + void *map_data; + struct module *module; + void *context; + void (*cont)(const struct firmware *fw, void *context); +}; + #define to_fwbuf(d) container_of(d, struct firmware_buf, ref) #define FW_LOADER_NO_CACHE 0 @@ -249,6 +272,9 @@ (unsigned int)buf->size); list_del(&buf->list); +#ifdef CONFIG_FW_LOADER_USER_HELPER + list_del(&buf->pending_list); +#endif spin_unlock(&fwc->lock); #ifdef CONFIG_FW_LOADER_USER_HELPER @@ -268,6 +294,11 @@ static void fw_free_buf(struct firmware_buf *buf) { struct firmware_cache *fwc = buf->fwc; + if (!fwc) { + kfree_const(buf->fw_id); + kfree(buf); + return; + } spin_lock(&fwc->lock); if (!kref_put(&buf->ref, __fw_free_buf)) spin_unlock(&fwc->lock); @@ -280,7 +311,10 @@ "/lib/firmware/updates/" UTS_RELEASE, "/lib/firmware/updates", "/lib/firmware/" UTS_RELEASE, - "/lib/firmware" + "/lib/firmware", + "/lib64/firmware", + "/usr/lib/hotplug/firmware", + "/firmware/image" }; /* @@ -302,6 +336,13 @@ size = i_size_read(file_inode(file)); if (size <= 0) return -EINVAL; + if (fw_buf->dest_size > 0 && fw_buf->dest_size < size) + return -EINVAL; + + if (fw_buf->dest_addr) + buf = fw_buf->map_fw_mem(fw_buf->dest_addr, + fw_buf->dest_size, fw_buf->map_data); + else buf = vmalloc(size); if (!buf) return -ENOMEM; @@ -316,14 +357,20 @@ goto fail; fw_buf->data = buf; fw_buf->size = size; + if (fw_buf->dest_addr) + fw_buf->unmap_fw_mem(buf, fw_buf->size, fw_buf->map_data); return 0; fail: + if (fw_buf->dest_addr) + fw_buf->unmap_fw_mem(buf, fw_buf->size, fw_buf->map_data); + else vfree(buf); return rc; } static int fw_get_filesystem_firmware(struct device *device, - struct firmware_buf *buf) + struct firmware_buf *buf, + phys_addr_t dest_addr, size_t dest_size) { int i, len; int rc = -ENOENT; @@ -658,6 +705,10 @@ case 1: /* discarding any previous partial load */ if (!test_bit(FW_STATUS_DONE, &fw_buf->status)) { + if (fw_buf->dest_addr) { + set_bit(FW_STATUS_LOADING, &fw_buf->status); + break; + } for (i = 0; i < fw_buf->nr_pages; i++) __free_page(fw_buf->pages[i]); kfree(fw_buf->pages); @@ -715,6 +766,102 @@ static DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store); +static int __firmware_data_rw(struct firmware_priv *fw_priv, char *buffer, + loff_t *offset, size_t count, int read) +{ + u8 __iomem *fw_buf; + struct firmware_buf *buf = fw_priv->buf; + int retval = count; + + if ((*offset + count) > buf->dest_size) { + pr_debug("%s: Failed size check.\n", __func__); + retval = -EINVAL; + goto out; + } + + fw_buf = buf->map_fw_mem(buf->dest_addr + *offset, count, + buf->map_data); + if (!fw_buf) { + pr_debug("%s: Failed ioremap.\n", __func__); + retval = -ENOMEM; + goto out; + } + + if (read) + memcpy(buffer, fw_buf, count); + else + memcpy(fw_buf, buffer, count); + + *offset += count; + buf->unmap_fw_mem(fw_buf, count, buf->map_data); + +out: + return retval; +} + +static ssize_t firmware_direct_read(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buffer, loff_t offset, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct firmware_priv *fw_priv = to_firmware_priv(dev); + struct firmware *fw; + ssize_t ret_count; + + if (!fw_priv->fw) + return -ENODEV; + + mutex_lock(&fw_lock); + fw = fw_priv->fw; + + if (offset > fw->size) { + ret_count = 0; + goto out; + } + if (count > fw->size - offset) + count = fw->size - offset; + + if (test_bit(FW_STATUS_DONE, &fw_priv->buf->status)) { + ret_count = -ENODEV; + goto out; + } + + ret_count = __firmware_data_rw(fw_priv, buffer, &offset, count, 1); +out: + mutex_unlock(&fw_lock); + return ret_count; +} + +static ssize_t firmware_direct_write(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buffer, loff_t offset, size_t count) +{ + struct device *dev = kobj_to_dev(kobj); + struct firmware_priv *fw_priv = to_firmware_priv(dev); + struct firmware *fw; + ssize_t retval; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + + mutex_lock(&fw_lock); + fw = fw_priv->fw; + if (!fw || !fw_priv->buf || + test_bit(FW_STATUS_DONE, &fw_priv->buf->status)) { + retval = -ENODEV; + goto out; + } + + retval = __firmware_data_rw(fw_priv, buffer, &offset, count, 0); + if (retval < 0) + goto out; + + fw_priv->buf->size = max_t(size_t, offset, fw_priv->buf->size); +out: + mutex_unlock(&fw_lock); + return retval; +} + static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buffer, loff_t offset, size_t count) @@ -864,6 +1011,13 @@ .write = firmware_data_write, }; +static struct bin_attribute firmware_direct_attr_data = { + .attr = { .name = "data", .mode = 0644 }, + .size = 0, + .read = firmware_direct_read, + .write = firmware_direct_write, +}; + static struct attribute *fw_dev_attrs[] = { &dev_attr_loading.attr, NULL @@ -879,31 +1033,47 @@ .bin_attrs = fw_dev_bin_attrs, }; +static struct bin_attribute *fw_dev_direct_bin_attrs[] = { + &firmware_direct_attr_data, + NULL +}; + +static const struct attribute_group fw_dev_direct_attr_group = { + .attrs = fw_dev_attrs, + .bin_attrs = fw_dev_direct_bin_attrs, +}; + static const struct attribute_group *fw_dev_attr_groups[] = { &fw_dev_attr_group, NULL }; +static const struct attribute_group *fw_dev_direct_attr_groups[] = { + &fw_dev_direct_attr_group, + NULL +}; + static struct firmware_priv * -fw_create_instance(struct firmware *firmware, const char *fw_name, - struct device *device, unsigned int opt_flags) +fw_create_instance(struct firmware *firmware, struct fw_desc *desc) { struct firmware_priv *fw_priv; struct device *f_dev; fw_priv = kzalloc(sizeof(*fw_priv), GFP_KERNEL); if (!fw_priv) { + dev_err(desc->device, "%s: kmalloc failed\n", __func__); fw_priv = ERR_PTR(-ENOMEM); goto exit; } - fw_priv->nowait = !!(opt_flags & FW_OPT_NOWAIT); + fw_priv->nowait = !!(desc->opt_flags & FW_OPT_NOWAIT); fw_priv->fw = firmware; + f_dev = &fw_priv->dev; device_initialize(f_dev); - dev_set_name(f_dev, "%s", fw_name); - f_dev->parent = device; + dev_set_name(f_dev, "%s", desc->name); + f_dev->parent = desc->device; f_dev->class = &firmware_class; f_dev->groups = fw_dev_attr_groups; exit: @@ -919,7 +1089,10 @@ struct firmware_buf *buf = fw_priv->buf; /* fall back on userspace loading */ - buf->is_paged_buf = true; + buf->is_paged_buf = buf->dest_addr ? false : true; + + if (buf->dest_addr) + f_dev->groups = fw_dev_direct_attr_groups; dev_set_uevent_suppress(f_dev, true); @@ -942,7 +1115,7 @@ timeout = MAX_JIFFY_OFFSET; } - timeout = wait_for_completion_interruptible_timeout(&buf->completion, + timeout = wait_for_completion_killable_timeout(&buf->completion, timeout); if (timeout == -ERESTARTSYS || !timeout) { retval = timeout; @@ -955,7 +1128,7 @@ if (is_fw_load_aborted(buf)) retval = -EAGAIN; - else if (!buf->data) + else if (!buf->data && buf->is_paged_buf) retval = -ENOMEM; device_del(f_dev); @@ -965,17 +1138,16 @@ } static int fw_load_from_user_helper(struct firmware *firmware, - const char *name, struct device *device, - unsigned int opt_flags, long timeout) + struct fw_desc *desc, long timeout) { struct firmware_priv *fw_priv; - fw_priv = fw_create_instance(firmware, name, device, opt_flags); + fw_priv = fw_create_instance(firmware, desc); if (IS_ERR(fw_priv)) return PTR_ERR(fw_priv); fw_priv->buf = firmware->priv; - return _request_firmware_load(fw_priv, opt_flags, timeout); + return _request_firmware_load(fw_priv, desc->opt_flags, timeout); } #ifdef CONFIG_PM_SLEEP @@ -996,9 +1168,8 @@ #else /* CONFIG_FW_LOADER_USER_HELPER */ static inline int -fw_load_from_user_helper(struct firmware *firmware, const char *name, - struct device *device, unsigned int opt_flags, - long timeout) +fw_load_from_user_helper(struct firmware *firmware, + struct fw_desc *desc, long timeout) { return -ENOENT; } @@ -1037,8 +1208,7 @@ * or a negative error code */ static int -_request_firmware_prepare(struct firmware **firmware_p, const char *name, - struct device *device) +_request_firmware_prepare(struct firmware **firmware_p, struct fw_desc *desc) { struct firmware *firmware; struct firmware_buf *buf; @@ -1046,17 +1216,31 @@ *firmware_p = firmware = kzalloc(sizeof(*firmware), GFP_KERNEL); if (!firmware) { - dev_err(device, "%s: kmalloc(struct firmware) failed\n", + dev_err(desc->device, "%s: kmalloc(struct firmware) failed\n", __func__); return -ENOMEM; } - if (fw_get_builtin_firmware(firmware, name)) { - dev_dbg(device, "firmware: using built-in firmware %s\n", name); + if (fw_get_builtin_firmware(firmware, desc->name)) { + dev_dbg(desc->device, "firmware: using built-in firmware %s\n", + desc->name); return 0; /* assigned */ } - ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf); + if (desc->opt_flags & FW_OPT_NOCACHE) { + buf = __allocate_fw_buf(desc->name, NULL); + if (!buf) + return -ENOMEM; + buf->dest_addr = desc->dest_addr; + buf->dest_size = desc->dest_size; + buf->map_fw_mem = desc->map_fw_mem; + buf->unmap_fw_mem = desc->unmap_fw_mem; + buf->map_data = desc->map_data; + firmware->priv = buf; + return 1; + } + + ret = fw_lookup_and_allocate_buf(desc->name, &fw_cache, &buf); /* * bind with 'buf' now to avoid warning in failure path @@ -1095,15 +1279,19 @@ * device may has been deleted already, but the problem * should be fixed in devres or driver core. */ - /* don't cache firmware handled without uevent */ - if (device && (opt_flags & FW_OPT_UEVENT)) + /* don't cache firmware handled without uevent, or when explicitly + * disabled + */ + if (device && (opt_flags & FW_OPT_UEVENT) + && !(opt_flags & FW_OPT_NOCACHE)) fw_add_devm_name(device, buf->fw_id); /* * After caching firmware image is started, let it piggyback * on request firmware. */ - if (buf->fwc->state == FW_LOADER_START_CACHE) { + if (!(opt_flags & FW_OPT_NOCACHE) + && (buf->fwc->state == FW_LOADER_START_CACHE)) { if (fw_cache_piggyback_on_request(buf->fw_id)) kref_get(&buf->ref); } @@ -1115,58 +1303,56 @@ } /* called from request_firmware() and request_firmware_work_func() */ -static int -_request_firmware(const struct firmware **firmware_p, const char *name, - struct device *device, unsigned int opt_flags) +static int _request_firmware(struct fw_desc *desc) { struct firmware *fw; long timeout; int ret; - if (!firmware_p) + if (!desc->firmware_p) return -EINVAL; - if (!name || name[0] == '\0') + if (!desc->name || desc->name[0] == '\0') return -EINVAL; - ret = _request_firmware_prepare(&fw, name, device); + ret = _request_firmware_prepare(&fw, desc); if (ret <= 0) /* error or already assigned */ goto out; ret = 0; timeout = firmware_loading_timeout(); - if (opt_flags & FW_OPT_NOWAIT) { + if (desc->opt_flags & FW_OPT_NOWAIT) { timeout = usermodehelper_read_lock_wait(timeout); if (!timeout) { - dev_dbg(device, "firmware: %s loading timed out\n", - name); + dev_dbg(desc->device, "firmware: %s loading timed out\n", + desc->name); ret = -EBUSY; goto out; } } else { ret = usermodehelper_read_trylock(); if (WARN_ON(ret)) { - dev_err(device, "firmware: %s will not be loaded\n", - name); + dev_err(desc->device, "firmware: %s will not be loaded\n", + desc->name); goto out; } } - ret = fw_get_filesystem_firmware(device, fw->priv); + ret = fw_get_filesystem_firmware(desc->device, fw->priv, + desc->dest_addr, desc->dest_size); if (ret) { - if (!(opt_flags & FW_OPT_NO_WARN)) - dev_warn(device, + if (!(desc->opt_flags & FW_OPT_NO_WARN)) + dev_dbg(desc->device, "Direct firmware load for %s failed with error %d\n", - name, ret); - if (opt_flags & FW_OPT_USERHELPER) { - dev_warn(device, "Falling back to user helper\n"); - ret = fw_load_from_user_helper(fw, name, device, - opt_flags, timeout); + desc->name, ret); + if (desc->opt_flags & FW_OPT_USERHELPER) { + dev_dbg(desc->device, "Falling back to user helper\n"); + ret = fw_load_from_user_helper(fw, desc, timeout); } } if (!ret) - ret = assign_firmware_buf(fw, device, opt_flags); + ret = assign_firmware_buf(fw, desc->device, desc->opt_flags); usermodehelper_read_unlock(); @@ -1176,7 +1362,7 @@ fw = NULL; } - *firmware_p = fw; + *desc->firmware_p = fw; return ret; } @@ -1204,13 +1390,21 @@ request_firmware(const struct firmware **firmware_p, const char *name, struct device *device) { + struct fw_desc desc; int ret; + desc.firmware_p = firmware_p; + desc.name = name; + desc.device = device; + desc.dest_addr = 0; + desc.dest_size = 0; + desc.opt_flags = FW_OPT_UEVENT | FW_OPT_FALLBACK; + /* Need to pin this module until return */ __module_get(THIS_MODULE); - ret = _request_firmware(firmware_p, name, device, - FW_OPT_UEVENT | FW_OPT_FALLBACK); + ret = _request_firmware(&desc); module_put(THIS_MODULE); + return ret; } EXPORT_SYMBOL(request_firmware); @@ -1229,17 +1423,70 @@ int request_firmware_direct(const struct firmware **firmware_p, const char *name, struct device *device) { + struct fw_desc desc; int ret; + desc.firmware_p = firmware_p; + desc.name = name; + desc.device = device; + desc.opt_flags = FW_OPT_UEVENT | FW_OPT_NO_WARN; + + /* Need to pin this module until return */ __module_get(THIS_MODULE); - ret = _request_firmware(firmware_p, name, device, - FW_OPT_UEVENT | FW_OPT_NO_WARN); + ret = _request_firmware(&desc); module_put(THIS_MODULE); + return ret; } EXPORT_SYMBOL_GPL(request_firmware_direct); /** + * request_firmware_into_buf: - send firmware request and wait for it + * @dest_addr: Destination address for the firmware + * @dest_size: Size of destination buffer + * + * Similar to request_firmware, except takes in a buffer address and + * copies firmware data directly to that buffer. Returns the size of + * the firmware that was loaded at dest_addr. This API prevents the + * caching of images. +*/ +int +request_firmware_into_buf(const char *name, struct device *device, + phys_addr_t dest_addr, size_t dest_size, + void * (*map_fw_mem)(phys_addr_t phys, size_t size, + void *data), + void (*unmap_fw_mem)(void *virt, size_t sz, void *data), + void *map_data) +{ + struct fw_desc desc; + const struct firmware *fp = NULL; + int ret; + + if (dest_addr && !map_fw_mem) + return -EINVAL; + if (dest_addr && dest_size <= 0) + return -EINVAL; + + desc.firmware_p = &fp; + desc.name = name; + desc.device = device; + desc.opt_flags = FW_OPT_FALLBACK | FW_OPT_UEVENT | FW_OPT_NOCACHE; + desc.dest_addr = dest_addr; + desc.dest_size = dest_size; + desc.map_fw_mem = map_fw_mem; + desc.unmap_fw_mem = unmap_fw_mem; + desc.map_data = map_data; + + ret = _request_firmware(&desc); + if (ret) + return ret; + ret = fp->size; + release_firmware(fp); + return ret; +} +EXPORT_SYMBOL_GPL(request_firmware_into_buf); + +/** * release_firmware: - release the resource associated with a firmware image * @fw: firmware resource to release **/ @@ -1254,31 +1501,70 @@ EXPORT_SYMBOL(release_firmware); /* Async support */ -struct firmware_work { - struct work_struct work; - struct module *module; - const char *name; - struct device *device; - void *context; - void (*cont)(const struct firmware *fw, void *context); - unsigned int opt_flags; -}; - static void request_firmware_work_func(struct work_struct *work) { - struct firmware_work *fw_work; const struct firmware *fw; + struct fw_desc *desc; + + desc = container_of(work, struct fw_desc, work); + desc->firmware_p = &fw; + _request_firmware(desc); + desc->cont(fw, desc->context); + put_device(desc->device); /* taken in request_firmware_nowait() */ + + module_put(desc->module); + kfree(desc); +} + +int +_request_firmware_nowait( + struct module *module, bool uevent, + const char *name, struct device *device, gfp_t gfp, void *context, + void (*cont)(const struct firmware *fw, void *context), + bool nocache, phys_addr_t dest_addr, size_t dest_size, + void * (*map_fw_mem)(phys_addr_t phys, size_t size, void *data), + void (*unmap_fw_mem)(void *virt, size_t size, void *data), + void *map_data) +{ + struct fw_desc *desc; + + if (dest_addr && !map_fw_mem) + return -EINVAL; + if (dest_addr && dest_size <= 0) + return -EINVAL; + + desc = kzalloc(sizeof(struct fw_desc), gfp); + if (!desc) + return -ENOMEM; + + desc->module = module; + desc->name = name; + desc->device = device; + desc->context = context; + desc->cont = cont; + desc->dest_addr = dest_addr; + desc->dest_size = dest_size; + desc->map_fw_mem = map_fw_mem; + desc->unmap_fw_mem = unmap_fw_mem; + desc->map_data = map_data; + desc->opt_flags = FW_OPT_FALLBACK | FW_OPT_NOWAIT; + + if (uevent) + desc->opt_flags |= FW_OPT_UEVENT; + else + desc->opt_flags |= FW_OPT_USERHELPER; + if (nocache) + desc->opt_flags |= FW_OPT_NOCACHE; - fw_work = container_of(work, struct firmware_work, work); + if (!try_module_get(module)) { + kfree(desc); + return -EFAULT; + } - _request_firmware(&fw, fw_work->name, fw_work->device, - fw_work->opt_flags); - fw_work->cont(fw, fw_work->context); - put_device(fw_work->device); /* taken in request_firmware_nowait() */ - - module_put(fw_work->module); - kfree_const(fw_work->name); - kfree(fw_work); + get_device(desc->device); + INIT_WORK(&desc->work, request_firmware_work_func); + schedule_work(&desc->work); + return 0; } /** @@ -1310,36 +1596,37 @@ const char *name, struct device *device, gfp_t gfp, void *context, void (*cont)(const struct firmware *fw, void *context)) { - struct firmware_work *fw_work; - - fw_work = kzalloc(sizeof(struct firmware_work), gfp); - if (!fw_work) - return -ENOMEM; - - fw_work->module = module; - fw_work->name = kstrdup_const(name, gfp); - if (!fw_work->name) { - kfree(fw_work); - return -ENOMEM; - } - fw_work->device = device; - fw_work->context = context; - fw_work->cont = cont; - fw_work->opt_flags = FW_OPT_NOWAIT | FW_OPT_FALLBACK | - (uevent ? FW_OPT_UEVENT : FW_OPT_USERHELPER); - - if (!try_module_get(module)) { - kfree_const(fw_work->name); - kfree(fw_work); - return -EFAULT; + return _request_firmware_nowait(module, uevent, name, device, gfp, + context, cont, false, 0, 0, NULL, NULL, NULL); } +EXPORT_SYMBOL(request_firmware_nowait); - get_device(fw_work->device); - INIT_WORK(&fw_work->work, request_firmware_work_func); - schedule_work(&fw_work->work); - return 0; +/** + * request_firmware_nowait_into_buf - asynchronous version of request_firmware + * @dest_addr: Destination address for the firmware + * @dest_size: Size of destination buffer + * + * Similar to request_firmware_nowait, except loads the firmware + * directly to a destination address without using an intermediate + * buffer. + * + **/ +int +request_firmware_nowait_into_buf( + struct module *module, bool uevent, + const char *name, struct device *device, gfp_t gfp, void *context, + void (*cont)(const struct firmware *fw, void *context), + phys_addr_t dest_addr, size_t dest_size, + void * (*map_fw_mem)(phys_addr_t phys, size_t size, void *data), + void (*unmap_fw_mem)(void *virt, size_t size, void *data), + void *map_data) +{ + return _request_firmware_nowait(module, uevent, name, device, gfp, + context, cont, true, dest_addr, + dest_size, map_fw_mem, unmap_fw_mem, + map_data); } -EXPORT_SYMBOL(request_firmware_nowait); +EXPORT_SYMBOL_GPL(request_firmware_nowait_into_buf); #ifdef CONFIG_PM_SLEEP static ASYNC_DOMAIN_EXCLUSIVE(fw_cache_domain); diff -ruw linux-4.4.115/drivers/base/Makefile linux-4.4.115-fbx/drivers/base/Makefile --- linux-4.4.115/drivers/base/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/Makefile 2019-01-22 16:16:22.871240683 +0100 @@ -9,7 +9,7 @@ obj-$(CONFIG_DMA_CMA) += dma-contiguous.o obj-y += power/ obj-$(CONFIG_HAS_DMA) += dma-mapping.o -obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o +obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o dma-removed.o obj-$(CONFIG_ISA) += isa.o obj-$(CONFIG_FW_LOADER) += firmware_class.o obj-$(CONFIG_NUMA) += node.o diff -ruw linux-4.4.115/drivers/base/platform.c linux-4.4.115-fbx/drivers/base/platform.c --- linux-4.4.115/drivers/base/platform.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/platform.c 2019-01-22 16:16:22.875240720 +0100 @@ -117,6 +117,26 @@ EXPORT_SYMBOL_GPL(platform_get_irq); /** + * platform_irq_count - Count the number of IRQs a platform device uses + * @dev: platform device + * + * Return: Number of IRQs a platform device uses or EPROBE_DEFER + */ +int platform_irq_count(struct platform_device *dev) +{ + int ret, nr = 0; + + while ((ret = platform_get_irq(dev, nr)) >= 0) + nr++; + + if (ret == -EPROBE_DEFER) + return ret; + + return nr; +} +EXPORT_SYMBOL_GPL(platform_irq_count); + +/** * platform_get_resource_byname - get a resource for a device by name * @dev: platform device * @type: resource type diff -ruw linux-4.4.115/drivers/base/power/main.c linux-4.4.115-fbx/drivers/base/power/main.c --- linux-4.4.115/drivers/base/power/main.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/main.c 2019-10-29 09:26:23.413200927 +0100 @@ -33,6 +33,7 @@ #include #include #include +#include #include "../base.h" #include "power.h" @@ -125,6 +126,7 @@ { pr_debug("PM: Adding info for %s:%s\n", dev->bus ? dev->bus->name : "No Bus", dev_name(dev)); + device_pm_check_callbacks(dev); mutex_lock(&dpm_list_mtx); if (dev->parent && dev->parent->power.is_prepared) dev_warn(dev, "parent %s should not be sleeping\n", @@ -147,6 +149,7 @@ mutex_unlock(&dpm_list_mtx); device_wakeup_disable(dev); pm_runtime_remove(dev); + device_pm_check_callbacks(dev); } /** @@ -1348,6 +1351,7 @@ pm_callback_t callback = NULL; char *info = NULL; int error = 0; + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; DECLARE_DPM_WATCHDOG_ON_STACK(wd); TRACE_DEVICE(dev); @@ -1368,6 +1372,9 @@ pm_wakeup_event(dev, 0); if (pm_wakeup_pending()) { + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); async_error = -EBUSY; goto Complete; } @@ -1570,6 +1577,11 @@ dev->power.wakeup_path = device_may_wakeup(dev); + if (dev->power.no_pm_callbacks) { + ret = 1; /* Let device go direct_complete */ + goto unlock; + } + if (dev->pm_domain) { info = "preparing power domain "; callback = dev->pm_domain->ops.prepare; @@ -1592,6 +1604,7 @@ if (callback) ret = callback(dev); +unlock: device_unlock(dev); if (ret < 0) { @@ -1720,3 +1733,30 @@ device_pm_unlock(); } EXPORT_SYMBOL_GPL(dpm_for_each_dev); + +static bool pm_ops_is_empty(const struct dev_pm_ops *ops) +{ + if (!ops) + return true; + + return !ops->prepare && + !ops->suspend && + !ops->suspend_late && + !ops->suspend_noirq && + !ops->resume_noirq && + !ops->resume_early && + !ops->resume && + !ops->complete; +} + +void device_pm_check_callbacks(struct device *dev) +{ + spin_lock_irq(&dev->power.lock); + dev->power.no_pm_callbacks = + (!dev->bus || pm_ops_is_empty(dev->bus->pm)) && + (!dev->class || pm_ops_is_empty(dev->class->pm)) && + (!dev->type || pm_ops_is_empty(dev->type->pm)) && + (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) && + (!dev->driver || pm_ops_is_empty(dev->driver->pm)); + spin_unlock_irq(&dev->power.lock); +} diff -ruw linux-4.4.115/drivers/base/power/opp/core.c linux-4.4.115-fbx/drivers/base/power/opp/core.c --- linux-4.4.115/drivers/base/power/opp/core.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/opp/core.c 2019-01-22 16:16:22.879240756 +0100 @@ -13,50 +13,52 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include #include #include #include +#include #include "opp.h" /* - * The root of the list of all devices. All device_opp structures branch off - * from here, with each device_opp containing the list of opp it supports in + * The root of the list of all opp-tables. All opp_table structures branch off + * from here, with each opp_table containing the list of opps it supports in * various states of availability. */ -static LIST_HEAD(dev_opp_list); +static LIST_HEAD(opp_tables); /* Lock to allow exclusive modification to the device and opp lists */ -DEFINE_MUTEX(dev_opp_list_lock); +DEFINE_MUTEX(opp_table_lock); #define opp_rcu_lockdep_assert() \ do { \ RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \ - !lockdep_is_held(&dev_opp_list_lock), \ + !lockdep_is_held(&opp_table_lock), \ "Missing rcu_read_lock() or " \ - "dev_opp_list_lock protection"); \ + "opp_table_lock protection"); \ } while (0) -static struct device_list_opp *_find_list_dev(const struct device *dev, - struct device_opp *dev_opp) +static struct opp_device *_find_opp_dev(const struct device *dev, + struct opp_table *opp_table) { - struct device_list_opp *list_dev; + struct opp_device *opp_dev; - list_for_each_entry(list_dev, &dev_opp->dev_list, node) - if (list_dev->dev == dev) - return list_dev; + list_for_each_entry(opp_dev, &opp_table->dev_list, node) + if (opp_dev->dev == dev) + return opp_dev; return NULL; } -static struct device_opp *_managed_opp(const struct device_node *np) +static struct opp_table *_managed_opp(const struct device_node *np) { - struct device_opp *dev_opp; + struct opp_table *opp_table; - list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) { - if (dev_opp->np == np) { + list_for_each_entry_rcu(opp_table, &opp_tables, node) { + if (opp_table->np == np) { /* * Multiple devices can point to the same OPP table and * so will have same node-pointer, np. @@ -64,7 +66,7 @@ * But the OPPs will be considered as shared only if the * OPP table contains a "opp-shared" property. */ - return dev_opp->shared_opp ? dev_opp : NULL; + return opp_table->shared_opp ? opp_table : NULL; } } @@ -72,24 +74,24 @@ } /** - * _find_device_opp() - find device_opp struct using device pointer - * @dev: device pointer used to lookup device OPPs + * _find_opp_table() - find opp_table struct using device pointer + * @dev: device pointer used to lookup OPP table * - * Search list of device OPPs for one containing matching device. Does a RCU - * reader operation to grab the pointer needed. + * Search OPP table for one containing matching device. Does a RCU reader + * operation to grab the pointer needed. * - * Return: pointer to 'struct device_opp' if found, otherwise -ENODEV or + * Return: pointer to 'struct opp_table' if found, otherwise -ENODEV or * -EINVAL based on type of error. * * Locking: For readers, this function must be called under rcu_read_lock(). - * device_opp is a RCU protected pointer, which means that device_opp is valid + * opp_table is a RCU protected pointer, which means that opp_table is valid * as long as we are under RCU lock. * - * For Writers, this function must be called with dev_opp_list_lock held. + * For Writers, this function must be called with opp_table_lock held. */ -struct device_opp *_find_device_opp(struct device *dev) +struct opp_table *_find_opp_table(struct device *dev) { - struct device_opp *dev_opp; + struct opp_table *opp_table; opp_rcu_lockdep_assert(); @@ -98,9 +100,9 @@ return ERR_PTR(-EINVAL); } - list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) - if (_find_list_dev(dev, dev_opp)) - return dev_opp; + list_for_each_entry_rcu(opp_table, &opp_tables, node) + if (_find_opp_dev(dev, opp_table)) + return opp_table; return ERR_PTR(-ENODEV); } @@ -213,16 +215,16 @@ */ unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev) { - struct device_opp *dev_opp; + struct opp_table *opp_table; unsigned long clock_latency_ns; rcu_read_lock(); - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) clock_latency_ns = 0; else - clock_latency_ns = dev_opp->clock_latency_ns_max; + clock_latency_ns = opp_table->clock_latency_ns_max; rcu_read_unlock(); return clock_latency_ns; @@ -230,6 +232,82 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency); /** + * dev_pm_opp_get_max_volt_latency() - Get max voltage latency in nanoseconds + * @dev: device for which we do this operation + * + * Return: This function returns the max voltage latency in nanoseconds. + * + * Locking: This function takes rcu_read_lock(). + */ +unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev) +{ + struct opp_table *opp_table; + struct dev_pm_opp *opp; + struct regulator *reg; + unsigned long latency_ns = 0; + unsigned long min_uV = ~0, max_uV = 0; + int ret; + + rcu_read_lock(); + + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + rcu_read_unlock(); + return 0; + } + + reg = opp_table->regulator; + if (IS_ERR(reg)) { + /* Regulator may not be required for device */ + if (reg) + dev_err(dev, "%s: Invalid regulator (%ld)\n", __func__, + PTR_ERR(reg)); + rcu_read_unlock(); + return 0; + } + + list_for_each_entry_rcu(opp, &opp_table->opp_list, node) { + if (!opp->available) + continue; + + if (opp->u_volt_min < min_uV) + min_uV = opp->u_volt_min; + if (opp->u_volt_max > max_uV) + max_uV = opp->u_volt_max; + } + + rcu_read_unlock(); + + /* + * The caller needs to ensure that opp_table (and hence the regulator) + * isn't freed, while we are executing this routine. + */ + ret = regulator_set_voltage_time(reg, min_uV, max_uV); + if (ret > 0) + latency_ns = ret * 1000; + + return latency_ns; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_volt_latency); + +/** + * dev_pm_opp_get_max_transition_latency() - Get max transition latency in + * nanoseconds + * @dev: device for which we do this operation + * + * Return: This function returns the max transition latency, in nanoseconds, to + * switch from one OPP to other. + * + * Locking: This function takes rcu_read_lock(). + */ +unsigned long dev_pm_opp_get_max_transition_latency(struct device *dev) +{ + return dev_pm_opp_get_max_volt_latency(dev) + + dev_pm_opp_get_max_clock_latency(dev); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_transition_latency); + +/** * dev_pm_opp_get_suspend_opp() - Get suspend opp * @dev: device for which we do this operation * @@ -244,21 +322,21 @@ */ struct dev_pm_opp *dev_pm_opp_get_suspend_opp(struct device *dev) { - struct device_opp *dev_opp; + struct opp_table *opp_table; opp_rcu_lockdep_assert(); - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp) || !dev_opp->suspend_opp || - !dev_opp->suspend_opp->available) + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table) || !opp_table->suspend_opp || + !opp_table->suspend_opp->available) return NULL; - return dev_opp->suspend_opp; + return opp_table->suspend_opp; } EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp); /** - * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list + * dev_pm_opp_get_opp_count() - Get number of opps available in the opp table * @dev: device for which we do this operation * * Return: This function returns the number of available opps if there are any, @@ -268,21 +346,21 @@ */ int dev_pm_opp_get_opp_count(struct device *dev) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *temp_opp; int count = 0; rcu_read_lock(); - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) { - count = PTR_ERR(dev_opp); - dev_err(dev, "%s: device OPP not found (%d)\n", + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + count = PTR_ERR(opp_table); + dev_err(dev, "%s: OPP table not found (%d)\n", __func__, count); goto out_unlock; } - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + list_for_each_entry_rcu(temp_opp, &opp_table->opp_list, node) { if (temp_opp->available) count++; } @@ -299,7 +377,7 @@ * @freq: frequency to search for * @available: true/false - match for available opp * - * Return: Searches for exact match in the opp list and returns pointer to the + * Return: Searches for exact match in the opp table and returns pointer to the * matching opp if found, else returns ERR_PTR in case of error and should * be handled using IS_ERR. Error return values can be: * EINVAL: for bad pointer @@ -323,19 +401,20 @@ unsigned long freq, bool available) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); opp_rcu_lockdep_assert(); - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int r = PTR_ERR(dev_opp); - dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r); + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + int r = PTR_ERR(opp_table); + + dev_err(dev, "%s: OPP table not found (%d)\n", __func__, r); return ERR_PTR(r); } - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + list_for_each_entry_rcu(temp_opp, &opp_table->opp_list, node) { if (temp_opp->available == available && temp_opp->rate == freq) { opp = temp_opp; @@ -371,7 +450,7 @@ struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, unsigned long *freq) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); opp_rcu_lockdep_assert(); @@ -381,11 +460,11 @@ return ERR_PTR(-EINVAL); } - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) + return ERR_CAST(opp_table); - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + list_for_each_entry_rcu(temp_opp, &opp_table->opp_list, node) { if (temp_opp->available && temp_opp->rate >= *freq) { opp = temp_opp; *freq = opp->rate; @@ -421,7 +500,7 @@ struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, unsigned long *freq) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); opp_rcu_lockdep_assert(); @@ -431,11 +510,11 @@ return ERR_PTR(-EINVAL); } - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) + return ERR_CAST(opp_table); - list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) { + list_for_each_entry_rcu(temp_opp, &opp_table->opp_list, node) { if (temp_opp->available) { /* go to the next node, before choosing prev */ if (temp_opp->rate > *freq) @@ -451,116 +530,343 @@ } EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor); -/* List-dev Helpers */ -static void _kfree_list_dev_rcu(struct rcu_head *head) +/* + * The caller needs to ensure that opp_table (and hence the clk) isn't freed, + * while clk returned here is used. + */ +static struct clk *_get_opp_clk(struct device *dev) { - struct device_list_opp *list_dev; + struct opp_table *opp_table; + struct clk *clk; - list_dev = container_of(head, struct device_list_opp, rcu_head); - kfree_rcu(list_dev, rcu_head); + rcu_read_lock(); + + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + dev_err(dev, "%s: device opp doesn't exist\n", __func__); + clk = ERR_CAST(opp_table); + goto unlock; } -static void _remove_list_dev(struct device_list_opp *list_dev, - struct device_opp *dev_opp) + clk = opp_table->clk; + if (IS_ERR(clk)) + dev_err(dev, "%s: No clock available for the device\n", + __func__); + +unlock: + rcu_read_unlock(); + return clk; +} + +static int _set_opp_voltage(struct device *dev, struct regulator *reg, + unsigned long u_volt, unsigned long u_volt_min, + unsigned long u_volt_max) { - list_del(&list_dev->node); - call_srcu(&dev_opp->srcu_head.srcu, &list_dev->rcu_head, - _kfree_list_dev_rcu); + int ret; + + /* Regulator not available for device */ + if (IS_ERR(reg)) { + dev_dbg(dev, "%s: regulator not available: %ld\n", __func__, + PTR_ERR(reg)); + return 0; } -struct device_list_opp *_add_list_dev(const struct device *dev, - struct device_opp *dev_opp) + dev_dbg(dev, "%s: voltages (mV): %lu %lu %lu\n", __func__, u_volt_min, + u_volt, u_volt_max); + + ret = regulator_set_voltage_triplet(reg, u_volt_min, u_volt, + u_volt_max); + if (ret) + dev_err(dev, "%s: failed to set voltage (%lu %lu %lu mV): %d\n", + __func__, u_volt_min, u_volt, u_volt_max, ret); + + return ret; +} + +/** + * dev_pm_opp_set_rate() - Configure new OPP based on frequency + * @dev: device for which we do this operation + * @target_freq: frequency to achieve + * + * This configures the power-supplies and clock source to the levels specified + * by the OPP corresponding to the target_freq. + * + * Locking: This function takes rcu_read_lock(). + */ +int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { - struct device_list_opp *list_dev; + struct opp_table *opp_table; + struct dev_pm_opp *old_opp, *opp; + struct regulator *reg; + struct clk *clk; + unsigned long freq, old_freq; + unsigned long u_volt, u_volt_min, u_volt_max; + unsigned long ou_volt, ou_volt_min, ou_volt_max; + int ret; + + if (unlikely(!target_freq)) { + dev_err(dev, "%s: Invalid target frequency %lu\n", __func__, + target_freq); + return -EINVAL; + } - list_dev = kzalloc(sizeof(*list_dev), GFP_KERNEL); - if (!list_dev) + clk = _get_opp_clk(dev); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + freq = clk_round_rate(clk, target_freq); + if ((long)freq <= 0) + freq = target_freq; + + old_freq = clk_get_rate(clk); + + /* Return early if nothing to do */ + if (old_freq == freq) { + dev_dbg(dev, "%s: old/new frequencies (%lu Hz) are same, nothing to do\n", + __func__, freq); + return 0; + } + + rcu_read_lock(); + + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + dev_err(dev, "%s: device opp doesn't exist\n", __func__); + rcu_read_unlock(); + return PTR_ERR(opp_table); + } + + old_opp = dev_pm_opp_find_freq_ceil(dev, &old_freq); + if (!IS_ERR(old_opp)) { + ou_volt = old_opp->u_volt; + ou_volt_min = old_opp->u_volt_min; + ou_volt_max = old_opp->u_volt_max; + } else { + dev_err(dev, "%s: failed to find current OPP for freq %lu (%ld)\n", + __func__, old_freq, PTR_ERR(old_opp)); + } + + opp = dev_pm_opp_find_freq_ceil(dev, &freq); + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + dev_err(dev, "%s: failed to find OPP for freq %lu (%d)\n", + __func__, freq, ret); + rcu_read_unlock(); + return ret; + } + + u_volt = opp->u_volt; + u_volt_min = opp->u_volt_min; + u_volt_max = opp->u_volt_max; + + reg = opp_table->regulator; + + rcu_read_unlock(); + + /* Scaling up? Scale voltage before frequency */ + if (freq > old_freq) { + ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min, + u_volt_max); + if (ret) + goto restore_voltage; + } + + /* Change frequency */ + + dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n", + __func__, old_freq, freq); + + ret = clk_set_rate(clk, freq); + if (ret) { + dev_err(dev, "%s: failed to set clock rate: %d\n", __func__, + ret); + goto restore_voltage; + } + + /* Scaling down? Scale voltage after frequency */ + if (freq < old_freq) { + ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min, + u_volt_max); + if (ret) + goto restore_freq; + } + + return 0; + +restore_freq: + if (clk_set_rate(clk, old_freq)) + dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n", + __func__, old_freq); +restore_voltage: + /* This shouldn't harm even if the voltages weren't updated earlier */ + if (!IS_ERR(old_opp)) + _set_opp_voltage(dev, reg, ou_volt, ou_volt_min, ou_volt_max); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_rate); + +/* OPP-dev Helpers */ +static void _kfree_opp_dev_rcu(struct rcu_head *head) +{ + struct opp_device *opp_dev; + + opp_dev = container_of(head, struct opp_device, rcu_head); + kfree_rcu(opp_dev, rcu_head); +} + +static void _remove_opp_dev(struct opp_device *opp_dev, + struct opp_table *opp_table) +{ + opp_debug_unregister(opp_dev, opp_table); + list_del(&opp_dev->node); + call_srcu(&opp_table->srcu_head.srcu, &opp_dev->rcu_head, + _kfree_opp_dev_rcu); +} + +struct opp_device *_add_opp_dev(const struct device *dev, + struct opp_table *opp_table) +{ + struct opp_device *opp_dev; + int ret; + + opp_dev = kzalloc(sizeof(*opp_dev), GFP_KERNEL); + if (!opp_dev) return NULL; - /* Initialize list-dev */ - list_dev->dev = dev; - list_add_rcu(&list_dev->node, &dev_opp->dev_list); + /* Initialize opp-dev */ + opp_dev->dev = dev; + list_add_rcu(&opp_dev->node, &opp_table->dev_list); - return list_dev; + /* Create debugfs entries for the opp_table */ + ret = opp_debug_register(opp_dev, opp_table); + if (ret) + dev_err(dev, "%s: Failed to register opp debugfs (%d)\n", + __func__, ret); + + return opp_dev; } /** - * _add_device_opp() - Find device OPP table or allocate a new one + * _add_opp_table() - Find OPP table or allocate a new one * @dev: device for which we do this operation * * It tries to find an existing table first, if it couldn't find one, it * allocates a new OPP table and returns that. * - * Return: valid device_opp pointer if success, else NULL. + * Return: valid opp_table pointer if success, else NULL. */ -static struct device_opp *_add_device_opp(struct device *dev) +static struct opp_table *_add_opp_table(struct device *dev) { - struct device_opp *dev_opp; - struct device_list_opp *list_dev; + struct opp_table *opp_table; + struct opp_device *opp_dev; + struct device_node *np; + int ret; - /* Check for existing list for 'dev' first */ - dev_opp = _find_device_opp(dev); - if (!IS_ERR(dev_opp)) - return dev_opp; + /* Check for existing table for 'dev' first */ + opp_table = _find_opp_table(dev); + if (!IS_ERR(opp_table)) + return opp_table; /* - * Allocate a new device OPP table. In the infrequent case where a new + * Allocate a new OPP table. In the infrequent case where a new * device is needed to be added, we pay this penalty. */ - dev_opp = kzalloc(sizeof(*dev_opp), GFP_KERNEL); - if (!dev_opp) + opp_table = kzalloc(sizeof(*opp_table), GFP_KERNEL); + if (!opp_table) return NULL; - INIT_LIST_HEAD(&dev_opp->dev_list); + INIT_LIST_HEAD(&opp_table->dev_list); - list_dev = _add_list_dev(dev, dev_opp); - if (!list_dev) { - kfree(dev_opp); + opp_dev = _add_opp_dev(dev, opp_table); + if (!opp_dev) { + kfree(opp_table); return NULL; } - srcu_init_notifier_head(&dev_opp->srcu_head); - INIT_LIST_HEAD(&dev_opp->opp_list); + /* + * Only required for backward compatibility with v1 bindings, but isn't + * harmful for other cases. And so we do it unconditionally. + */ + np = of_node_get(dev->of_node); + if (np) { + u32 val; + + if (!of_property_read_u32(np, "clock-latency", &val)) + opp_table->clock_latency_ns_max = val; + of_property_read_u32(np, "voltage-tolerance", + &opp_table->voltage_tolerance_v1); + of_node_put(np); + } + + /* Set regulator to a non-NULL error value */ + opp_table->regulator = ERR_PTR(-ENXIO); - /* Secure the device list modification */ - list_add_rcu(&dev_opp->node, &dev_opp_list); - return dev_opp; + /* Find clk for the device */ + opp_table->clk = clk_get(dev, NULL); + if (IS_ERR(opp_table->clk)) { + ret = PTR_ERR(opp_table->clk); + if (ret != -EPROBE_DEFER) + dev_dbg(dev, "%s: Couldn't find clock: %d\n", __func__, + ret); + } + + srcu_init_notifier_head(&opp_table->srcu_head); + INIT_LIST_HEAD(&opp_table->opp_list); + + /* Secure the device table modification */ + list_add_rcu(&opp_table->node, &opp_tables); + return opp_table; } /** - * _kfree_device_rcu() - Free device_opp RCU handler + * _kfree_device_rcu() - Free opp_table RCU handler * @head: RCU head */ static void _kfree_device_rcu(struct rcu_head *head) { - struct device_opp *device_opp = container_of(head, struct device_opp, rcu_head); + struct opp_table *opp_table = container_of(head, struct opp_table, + rcu_head); - kfree_rcu(device_opp, rcu_head); + kfree_rcu(opp_table, rcu_head); } /** - * _remove_device_opp() - Removes a device OPP table - * @dev_opp: device OPP table to be removed. + * _remove_opp_table() - Removes a OPP table + * @opp_table: OPP table to be removed. * - * Removes/frees device OPP table it it doesn't contain any OPPs. + * Removes/frees OPP table if it doesn't contain any OPPs. */ -static void _remove_device_opp(struct device_opp *dev_opp) +static void _remove_opp_table(struct opp_table *opp_table) { - struct device_list_opp *list_dev; + struct opp_device *opp_dev; + + if (!list_empty(&opp_table->opp_list)) + return; + + if (opp_table->supported_hw) + return; - if (!list_empty(&dev_opp->opp_list)) + if (opp_table->prop_name) return; - list_dev = list_first_entry(&dev_opp->dev_list, struct device_list_opp, + if (!IS_ERR(opp_table->regulator)) + return; + + /* Release clk */ + if (!IS_ERR(opp_table->clk)) + clk_put(opp_table->clk); + + opp_dev = list_first_entry(&opp_table->dev_list, struct opp_device, node); - _remove_list_dev(list_dev, dev_opp); + _remove_opp_dev(opp_dev, opp_table); /* dev_list must be empty now */ - WARN_ON(!list_empty(&dev_opp->dev_list)); + WARN_ON(!list_empty(&opp_table->dev_list)); - list_del_rcu(&dev_opp->node); - call_srcu(&dev_opp->srcu_head.srcu, &dev_opp->rcu_head, + list_del_rcu(&opp_table->node); + call_srcu(&opp_table->srcu_head.srcu, &opp_table->rcu_head, _kfree_device_rcu); } @@ -577,17 +883,17 @@ /** * _opp_remove() - Remove an OPP from a table definition - * @dev_opp: points back to the device_opp struct this opp belongs to + * @opp_table: points back to the opp_table struct this opp belongs to * @opp: pointer to the OPP to remove * @notify: OPP_EVENT_REMOVE notification should be sent or not * - * This function removes an opp definition from the opp list. + * This function removes an opp definition from the opp table. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * It is assumed that the caller holds required mutex for an RCU updater * strategy. */ -static void _opp_remove(struct device_opp *dev_opp, +static void _opp_remove(struct opp_table *opp_table, struct dev_pm_opp *opp, bool notify) { /* @@ -595,21 +901,23 @@ * frequency/voltage list. */ if (notify) - srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_REMOVE, opp); + srcu_notifier_call_chain(&opp_table->srcu_head, + OPP_EVENT_REMOVE, opp); + opp_debug_remove_one(opp); list_del_rcu(&opp->node); - call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu); + call_srcu(&opp_table->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu); - _remove_device_opp(dev_opp); + _remove_opp_table(opp_table); } /** - * dev_pm_opp_remove() - Remove an OPP from OPP list + * dev_pm_opp_remove() - Remove an OPP from OPP table * @dev: device for which we do this operation * @freq: OPP to remove with matching 'freq' * - * This function removes an opp from the opp list. + * This function removes an opp from the opp table. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function internally uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where @@ -618,17 +926,17 @@ void dev_pm_opp_remove(struct device *dev, unsigned long freq) { struct dev_pm_opp *opp; - struct device_opp *dev_opp; + struct opp_table *opp_table; bool found = false; - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) goto unlock; - list_for_each_entry(opp, &dev_opp->opp_list, node) { + list_for_each_entry(opp, &opp_table->opp_list, node) { if (opp->rate == freq) { found = true; break; @@ -641,14 +949,14 @@ goto unlock; } - _opp_remove(dev_opp, opp, true); + _opp_remove(opp_table, opp, true); unlock: - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); } EXPORT_SYMBOL_GPL(dev_pm_opp_remove); static struct dev_pm_opp *_allocate_opp(struct device *dev, - struct device_opp **dev_opp) + struct opp_table **opp_table) { struct dev_pm_opp *opp; @@ -659,8 +967,8 @@ INIT_LIST_HEAD(&opp->node); - *dev_opp = _add_device_opp(dev); - if (!*dev_opp) { + *opp_table = _add_opp_table(dev); + if (!*opp_table) { kfree(opp); return NULL; } @@ -668,21 +976,38 @@ return opp; } +static bool _opp_supported_by_regulators(struct dev_pm_opp *opp, + struct opp_table *opp_table) +{ + struct regulator *reg = opp_table->regulator; + + if (!IS_ERR(reg) && + !regulator_is_supported_voltage(reg, opp->u_volt_min, + opp->u_volt_max)) { + pr_warn("%s: OPP minuV: %lu maxuV: %lu, not supported by regulator\n", + __func__, opp->u_volt_min, opp->u_volt_max); + return false; + } + + return true; +} + static int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, - struct device_opp *dev_opp) + struct opp_table *opp_table) { struct dev_pm_opp *opp; - struct list_head *head = &dev_opp->opp_list; + struct list_head *head = &opp_table->opp_list; + int ret; /* * Insert new OPP in order of increasing frequency and discard if * already present. * - * Need to use &dev_opp->opp_list in the condition part of the 'for' + * Need to use &opp_table->opp_list in the condition part of the 'for' * loop, don't replace it with head otherwise it will become an infinite * loop. */ - list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) { + list_for_each_entry_rcu(opp, &opp_table->opp_list, node) { if (new_opp->rate > opp->rate) { head = &opp->node; continue; @@ -700,9 +1025,20 @@ 0 : -EEXIST; } - new_opp->dev_opp = dev_opp; + new_opp->opp_table = opp_table; list_add_rcu(&new_opp->node, head); + ret = opp_debug_create_one(new_opp, opp_table); + if (ret) + dev_err(dev, "%s: Failed to register opp to debugfs (%d)\n", + __func__, ret); + + if (!_opp_supported_by_regulators(new_opp, opp_table)) { + new_opp->available = false; + dev_warn(dev, "%s: OPP not supported by regulators (%lu)\n", + __func__, new_opp->rate); + } + return 0; } @@ -713,14 +1049,14 @@ * @u_volt: Voltage in uVolts for this OPP * @dynamic: Dynamically added OPPs. * - * This function adds an opp definition to the opp list and returns status. + * This function adds an opp definition to the opp table and returns status. * The opp is made available by default and it can be controlled using * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove. * * NOTE: "dynamic" parameter impacts OPPs added by the dev_pm_opp_of_add_table * and freed by dev_pm_opp_of_remove_table. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function internally uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where @@ -736,14 +1072,15 @@ static int _opp_add_v1(struct device *dev, unsigned long freq, long u_volt, bool dynamic) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *new_opp; + unsigned long tol; int ret; - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); - new_opp = _allocate_opp(dev, &dev_opp); + new_opp = _allocate_opp(dev, &opp_table); if (!new_opp) { ret = -ENOMEM; goto unlock; @@ -751,60 +1088,77 @@ /* populate the opp table */ new_opp->rate = freq; + tol = u_volt * opp_table->voltage_tolerance_v1 / 100; new_opp->u_volt = u_volt; + new_opp->u_volt_min = u_volt - tol; + new_opp->u_volt_max = u_volt + tol; new_opp->available = true; new_opp->dynamic = dynamic; - ret = _opp_add(dev, new_opp, dev_opp); + ret = _opp_add(dev, new_opp, opp_table); if (ret) goto free_opp; - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); /* * Notify the changes in the availability of the operable * frequency/voltage list. */ - srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp); + srcu_notifier_call_chain(&opp_table->srcu_head, OPP_EVENT_ADD, new_opp); return 0; free_opp: - _opp_remove(dev_opp, new_opp, false); + _opp_remove(opp_table, new_opp, false); unlock: - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); return ret; } /* TODO: Support multiple regulators */ -static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev) +static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, + struct opp_table *opp_table) { u32 microvolt[3] = {0}; u32 val; int count, ret; + struct property *prop = NULL; + char name[NAME_MAX]; + + /* Search for "opp-microvolt-" */ + if (opp_table->prop_name) { + snprintf(name, sizeof(name), "opp-microvolt-%s", + opp_table->prop_name); + prop = of_find_property(opp->np, name, NULL); + } + + if (!prop) { + /* Search for "opp-microvolt" */ + sprintf(name, "opp-microvolt"); + prop = of_find_property(opp->np, name, NULL); /* Missing property isn't a problem, but an invalid entry is */ - if (!of_find_property(opp->np, "opp-microvolt", NULL)) + if (!prop) return 0; + } - count = of_property_count_u32_elems(opp->np, "opp-microvolt"); + count = of_property_count_u32_elems(opp->np, name); if (count < 0) { - dev_err(dev, "%s: Invalid opp-microvolt property (%d)\n", - __func__, count); + dev_err(dev, "%s: Invalid %s property (%d)\n", + __func__, name, count); return count; } /* There can be one or three elements here */ if (count != 1 && count != 3) { - dev_err(dev, "%s: Invalid number of elements in opp-microvolt property (%d)\n", - __func__, count); + dev_err(dev, "%s: Invalid number of elements in %s property (%d)\n", + __func__, name, count); return -EINVAL; } - ret = of_property_read_u32_array(opp->np, "opp-microvolt", microvolt, - count); + ret = of_property_read_u32_array(opp->np, name, microvolt, count); if (ret) { - dev_err(dev, "%s: error parsing opp-microvolt: %d\n", __func__, - ret); + dev_err(dev, "%s: error parsing %s: %d\n", __func__, name, ret); return -EINVAL; } @@ -818,22 +1172,391 @@ opp->u_volt_max = microvolt[2]; } - if (!of_property_read_u32(opp->np, "opp-microamp", &val)) + /* Search for "opp-microamp-" */ + prop = NULL; + if (opp_table->prop_name) { + snprintf(name, sizeof(name), "opp-microamp-%s", + opp_table->prop_name); + prop = of_find_property(opp->np, name, NULL); + } + + if (!prop) { + /* Search for "opp-microamp" */ + sprintf(name, "opp-microamp"); + prop = of_find_property(opp->np, name, NULL); + } + + if (prop && !of_property_read_u32(opp->np, name, &val)) opp->u_amp = val; return 0; } /** + * dev_pm_opp_set_supported_hw() - Set supported platforms + * @dev: Device for which supported-hw has to be set. + * @versions: Array of hierarchy of versions to match. + * @count: Number of elements in the array. + * + * This is required only for the V2 bindings, and it enables a platform to + * specify the hierarchy of versions it supports. OPP layer will then enable + * OPPs, which are available for those versions, based on its 'opp-supported-hw' + * property. + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +int dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, + unsigned int count) +{ + struct opp_table *opp_table; + int ret = 0; + + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); + + opp_table = _add_opp_table(dev); + if (!opp_table) { + ret = -ENOMEM; + goto unlock; + } + + /* Make sure there are no concurrent readers while updating opp_table */ + WARN_ON(!list_empty(&opp_table->opp_list)); + + /* Do we already have a version hierarchy associated with opp_table? */ + if (opp_table->supported_hw) { + dev_err(dev, "%s: Already have supported hardware list\n", + __func__); + ret = -EBUSY; + goto err; + } + + opp_table->supported_hw = kmemdup(versions, count * sizeof(*versions), + GFP_KERNEL); + if (!opp_table->supported_hw) { + ret = -ENOMEM; + goto err; + } + + opp_table->supported_hw_count = count; + mutex_unlock(&opp_table_lock); + return 0; + +err: + _remove_opp_table(opp_table); +unlock: + mutex_unlock(&opp_table_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_supported_hw); + +/** + * dev_pm_opp_put_supported_hw() - Releases resources blocked for supported hw + * @dev: Device for which supported-hw has to be put. + * + * This is required only for the V2 bindings, and is called for a matching + * dev_pm_opp_set_supported_hw(). Until this is called, the opp_table structure + * will not be freed. + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_put_supported_hw(struct device *dev) +{ + struct opp_table *opp_table; + + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); + + /* Check for existing table for 'dev' first */ + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + dev_err(dev, "Failed to find opp_table: %ld\n", + PTR_ERR(opp_table)); + goto unlock; + } + + /* Make sure there are no concurrent readers while updating opp_table */ + WARN_ON(!list_empty(&opp_table->opp_list)); + + if (!opp_table->supported_hw) { + dev_err(dev, "%s: Doesn't have supported hardware list\n", + __func__); + goto unlock; + } + + kfree(opp_table->supported_hw); + opp_table->supported_hw = NULL; + opp_table->supported_hw_count = 0; + + /* Try freeing opp_table if this was the last blocking resource */ + _remove_opp_table(opp_table); + +unlock: + mutex_unlock(&opp_table_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_put_supported_hw); + +/** + * dev_pm_opp_set_prop_name() - Set prop-extn name + * @dev: Device for which the prop-name has to be set. + * @name: name to postfix to properties. + * + * This is required only for the V2 bindings, and it enables a platform to + * specify the extn to be used for certain property names. The properties to + * which the extension will apply are opp-microvolt and opp-microamp. OPP core + * should postfix the property name with - while looking for them. + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +int dev_pm_opp_set_prop_name(struct device *dev, const char *name) +{ + struct opp_table *opp_table; + int ret = 0; + + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); + + opp_table = _add_opp_table(dev); + if (!opp_table) { + ret = -ENOMEM; + goto unlock; + } + + /* Make sure there are no concurrent readers while updating opp_table */ + WARN_ON(!list_empty(&opp_table->opp_list)); + + /* Do we already have a prop-name associated with opp_table? */ + if (opp_table->prop_name) { + dev_err(dev, "%s: Already have prop-name %s\n", __func__, + opp_table->prop_name); + ret = -EBUSY; + goto err; + } + + opp_table->prop_name = kstrdup(name, GFP_KERNEL); + if (!opp_table->prop_name) { + ret = -ENOMEM; + goto err; + } + + mutex_unlock(&opp_table_lock); + return 0; + +err: + _remove_opp_table(opp_table); +unlock: + mutex_unlock(&opp_table_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_prop_name); + +/** + * dev_pm_opp_put_prop_name() - Releases resources blocked for prop-name + * @dev: Device for which the prop-name has to be put. + * + * This is required only for the V2 bindings, and is called for a matching + * dev_pm_opp_set_prop_name(). Until this is called, the opp_table structure + * will not be freed. + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_put_prop_name(struct device *dev) +{ + struct opp_table *opp_table; + + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); + + /* Check for existing table for 'dev' first */ + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + dev_err(dev, "Failed to find opp_table: %ld\n", + PTR_ERR(opp_table)); + goto unlock; + } + + /* Make sure there are no concurrent readers while updating opp_table */ + WARN_ON(!list_empty(&opp_table->opp_list)); + + if (!opp_table->prop_name) { + dev_err(dev, "%s: Doesn't have a prop-name\n", __func__); + goto unlock; + } + + kfree(opp_table->prop_name); + opp_table->prop_name = NULL; + + /* Try freeing opp_table if this was the last blocking resource */ + _remove_opp_table(opp_table); + +unlock: + mutex_unlock(&opp_table_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); + +/** + * dev_pm_opp_set_regulator() - Set regulator name for the device + * @dev: Device for which regulator name is being set. + * @name: Name of the regulator. + * + * In order to support OPP switching, OPP layer needs to know the name of the + * device's regulator, as the core would be required to switch voltages as well. + * + * This must be called before any OPPs are initialized for the device. + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +int dev_pm_opp_set_regulator(struct device *dev, const char *name) +{ + struct opp_table *opp_table; + struct regulator *reg; + int ret; + + mutex_lock(&opp_table_lock); + + opp_table = _add_opp_table(dev); + if (!opp_table) { + ret = -ENOMEM; + goto unlock; + } + + /* This should be called before OPPs are initialized */ + if (WARN_ON(!list_empty(&opp_table->opp_list))) { + ret = -EBUSY; + goto err; + } + + /* Already have a regulator set */ + if (WARN_ON(!IS_ERR(opp_table->regulator))) { + ret = -EBUSY; + goto err; + } + /* Allocate the regulator */ + reg = regulator_get_optional(dev, name); + if (IS_ERR(reg)) { + ret = PTR_ERR(reg); + if (ret != -EPROBE_DEFER) + dev_err(dev, "%s: no regulator (%s) found: %d\n", + __func__, name, ret); + goto err; + } + + opp_table->regulator = reg; + + mutex_unlock(&opp_table_lock); + return 0; + +err: + _remove_opp_table(opp_table); +unlock: + mutex_unlock(&opp_table_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator); + +/** + * dev_pm_opp_put_regulator() - Releases resources blocked for regulator + * @dev: Device for which regulator was set. + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_put_regulator(struct device *dev) +{ + struct opp_table *opp_table; + + mutex_lock(&opp_table_lock); + + /* Check for existing table for 'dev' first */ + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + dev_err(dev, "Failed to find opp_table: %ld\n", + PTR_ERR(opp_table)); + goto unlock; + } + + if (IS_ERR(opp_table->regulator)) { + dev_err(dev, "%s: Doesn't have regulator set\n", __func__); + goto unlock; + } + + /* Make sure there are no concurrent readers while updating opp_table */ + WARN_ON(!list_empty(&opp_table->opp_list)); + + regulator_put(opp_table->regulator); + opp_table->regulator = ERR_PTR(-ENXIO); + + /* Try freeing opp_table if this was the last blocking resource */ + _remove_opp_table(opp_table); + +unlock: + mutex_unlock(&opp_table_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulator); + +static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table, + struct device_node *np) +{ + unsigned int count = opp_table->supported_hw_count; + u32 version; + int ret; + + if (!opp_table->supported_hw) + return true; + + while (count--) { + ret = of_property_read_u32_index(np, "opp-supported-hw", count, + &version); + if (ret) { + dev_warn(dev, "%s: failed to read opp-supported-hw property at index %d: %d\n", + __func__, count, ret); + return false; + } + + /* Both of these are bitwise masks of the versions */ + if (!(version & opp_table->supported_hw[count])) + return false; + } + + return true; +} + +/** * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings) * @dev: device for which we do this operation * @np: device node * - * This function adds an opp definition to the opp list and returns status. The + * This function adds an opp definition to the opp table and returns status. The * opp can be controlled using dev_pm_opp_enable/disable functions and may be * removed by dev_pm_opp_remove. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function internally uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where @@ -849,16 +1572,16 @@ */ static int _opp_add_static_v2(struct device *dev, struct device_node *np) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *new_opp; u64 rate; u32 val; int ret; - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); - new_opp = _allocate_opp(dev, &dev_opp); + new_opp = _allocate_opp(dev, &opp_table); if (!new_opp) { ret = -ENOMEM; goto unlock; @@ -870,6 +1593,12 @@ goto free_opp; } + /* Check if the OPP supports hardware's hierarchy of versions or not */ + if (!_opp_is_supported(dev, opp_table, np)) { + dev_dbg(dev, "OPP not supported by hardware: %llu\n", rate); + goto free_opp; + } + /* * Rate is defined as an unsigned long in clk API, and so casting * explicitly to its type. Must be fixed once rate is 64 bit @@ -885,28 +1614,30 @@ if (!of_property_read_u32(np, "clock-latency-ns", &val)) new_opp->clock_latency_ns = val; - ret = opp_parse_supplies(new_opp, dev); + ret = opp_parse_supplies(new_opp, dev, opp_table); if (ret) goto free_opp; - ret = _opp_add(dev, new_opp, dev_opp); + ret = _opp_add(dev, new_opp, opp_table); if (ret) goto free_opp; /* OPP to select on device suspend */ if (of_property_read_bool(np, "opp-suspend")) { - if (dev_opp->suspend_opp) + if (opp_table->suspend_opp) { dev_warn(dev, "%s: Multiple suspend OPPs found (%lu %lu)\n", - __func__, dev_opp->suspend_opp->rate, + __func__, opp_table->suspend_opp->rate, new_opp->rate); - else - dev_opp->suspend_opp = new_opp; + } else { + new_opp->suspend = true; + opp_table->suspend_opp = new_opp; + } } - if (new_opp->clock_latency_ns > dev_opp->clock_latency_ns_max) - dev_opp->clock_latency_ns_max = new_opp->clock_latency_ns; + if (new_opp->clock_latency_ns > opp_table->clock_latency_ns_max) + opp_table->clock_latency_ns_max = new_opp->clock_latency_ns; - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n", __func__, new_opp->turbo, new_opp->rate, new_opp->u_volt, @@ -917,13 +1648,13 @@ * Notify the changes in the availability of the operable * frequency/voltage list. */ - srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp); + srcu_notifier_call_chain(&opp_table->srcu_head, OPP_EVENT_ADD, new_opp); return 0; free_opp: - _opp_remove(dev_opp, new_opp, false); + _opp_remove(opp_table, new_opp, false); unlock: - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); return ret; } @@ -933,11 +1664,11 @@ * @freq: Frequency in Hz for this OPP * @u_volt: Voltage in uVolts for this OPP * - * This function adds an opp definition to the opp list and returns status. + * This function adds an opp definition to the opp table and returns status. * The opp is made available by default and it can be controlled using * dev_pm_opp_enable/disable functions. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function internally uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where @@ -969,7 +1700,7 @@ * copy operation, returns 0 if no modification was done OR modification was * successful. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function internally uses RCU updater strategy with mutex locks to * keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where @@ -978,7 +1709,7 @@ static int _opp_set_availability(struct device *dev, unsigned long freq, bool availability_req) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *new_opp, *tmp_opp, *opp = ERR_PTR(-ENODEV); int r = 0; @@ -987,18 +1718,18 @@ if (!new_opp) return -ENOMEM; - mutex_lock(&dev_opp_list_lock); + mutex_lock(&opp_table_lock); - /* Find the device_opp */ - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) { - r = PTR_ERR(dev_opp); + /* Find the opp_table */ + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + r = PTR_ERR(opp_table); dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r); goto unlock; } /* Do we have the frequency? */ - list_for_each_entry(tmp_opp, &dev_opp->opp_list, node) { + list_for_each_entry(tmp_opp, &opp_table->opp_list, node) { if (tmp_opp->rate == freq) { opp = tmp_opp; break; @@ -1019,21 +1750,21 @@ new_opp->available = availability_req; list_replace_rcu(&opp->node, &new_opp->node); - mutex_unlock(&dev_opp_list_lock); - call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu); + mutex_unlock(&opp_table_lock); + call_srcu(&opp_table->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu); /* Notify the change of the OPP availability */ if (availability_req) - srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ENABLE, - new_opp); + srcu_notifier_call_chain(&opp_table->srcu_head, + OPP_EVENT_ENABLE, new_opp); else - srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_DISABLE, - new_opp); + srcu_notifier_call_chain(&opp_table->srcu_head, + OPP_EVENT_DISABLE, new_opp); return 0; unlock: - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); kfree(new_opp); return r; } @@ -1047,7 +1778,7 @@ * corresponding error value. It is meant to be used for users an OPP available * after being temporarily made unavailable with dev_pm_opp_disable. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function indirectly uses RCU and mutex locks to keep the * integrity of the internal data structures. Callers should ensure that * this function is *NOT* called under RCU protection or in contexts where @@ -1073,7 +1804,7 @@ * control by users to make this OPP not available until the circumstances are * right to make it available again (with a call to dev_pm_opp_enable). * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function indirectly uses RCU and mutex locks to keep the * integrity of the internal data structures. Callers should ensure that * this function is *NOT* called under RCU protection or in contexts where @@ -1091,26 +1822,26 @@ /** * dev_pm_opp_get_notifier() - find notifier_head of the device with opp - * @dev: device pointer used to lookup device OPPs. + * @dev: device pointer used to lookup OPP table. * * Return: pointer to notifier head if found, otherwise -ENODEV or * -EINVAL based on type of error casted as pointer. value must be checked * with IS_ERR to determine valid pointer or error result. * - * Locking: This function must be called under rcu_read_lock(). dev_opp is a RCU - * protected pointer. The reason for the same is that the opp pointer which is - * returned will remain valid for use with opp_get_{voltage, freq} only while + * Locking: This function must be called under rcu_read_lock(). opp_table is a + * RCU protected pointer. The reason for the same is that the opp pointer which + * is returned will remain valid for use with opp_get_{voltage, freq} only while * under the locked area. The pointer returned must be used prior to unlocking * with rcu_read_unlock() to maintain the integrity of the pointer. */ struct srcu_notifier_head *dev_pm_opp_get_notifier(struct device *dev) { - struct device_opp *dev_opp = _find_device_opp(dev); + struct opp_table *opp_table = _find_opp_table(dev); - if (IS_ERR(dev_opp)) - return ERR_CAST(dev_opp); /* matching type */ + if (IS_ERR(opp_table)) + return ERR_CAST(opp_table); /* matching type */ - return &dev_opp->srcu_head; + return &opp_table->srcu_head; } EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier); @@ -1118,11 +1849,11 @@ /** * dev_pm_opp_of_remove_table() - Free OPP table entries created from static DT * entries - * @dev: device pointer used to lookup device OPPs. + * @dev: device pointer used to lookup OPP table. * * Free OPPs created using static entries present in DT. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function indirectly uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where @@ -1130,38 +1861,38 @@ */ void dev_pm_opp_of_remove_table(struct device *dev) { - struct device_opp *dev_opp; + struct opp_table *opp_table; struct dev_pm_opp *opp, *tmp; - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); + /* Hold our table modification lock here */ + mutex_lock(&opp_table_lock); - /* Check for existing list for 'dev' */ - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int error = PTR_ERR(dev_opp); + /* Check for existing table for 'dev' */ + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + int error = PTR_ERR(opp_table); if (error != -ENODEV) - WARN(1, "%s: dev_opp: %d\n", + WARN(1, "%s: opp_table: %d\n", IS_ERR_OR_NULL(dev) ? "Invalid device" : dev_name(dev), error); goto unlock; } - /* Find if dev_opp manages a single device */ - if (list_is_singular(&dev_opp->dev_list)) { + /* Find if opp_table manages a single device */ + if (list_is_singular(&opp_table->dev_list)) { /* Free static OPPs */ - list_for_each_entry_safe(opp, tmp, &dev_opp->opp_list, node) { + list_for_each_entry_safe(opp, tmp, &opp_table->opp_list, node) { if (!opp->dynamic) - _opp_remove(dev_opp, opp, true); + _opp_remove(opp_table, opp, true); } } else { - _remove_list_dev(_find_list_dev(dev, dev_opp), dev_opp); + _remove_opp_dev(_find_opp_dev(dev, opp_table), opp_table); } unlock: - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); } EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table); @@ -1182,22 +1913,22 @@ static int _of_add_opp_table_v2(struct device *dev, struct device_node *opp_np) { struct device_node *np; - struct device_opp *dev_opp; + struct opp_table *opp_table; int ret = 0, count = 0; - mutex_lock(&dev_opp_list_lock); + mutex_lock(&opp_table_lock); - dev_opp = _managed_opp(opp_np); - if (dev_opp) { + opp_table = _managed_opp(opp_np); + if (opp_table) { /* OPPs are already managed */ - if (!_add_list_dev(dev, dev_opp)) + if (!_add_opp_dev(dev, opp_table)) ret = -ENOMEM; - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); return ret; } - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); - /* We have opp-list node now, iterate over it and add OPPs */ + /* We have opp-table node now, iterate over it and add OPPs */ for_each_available_child_of_node(opp_np, np) { count++; @@ -1214,19 +1945,19 @@ if (WARN_ON(!count)) return -ENOENT; - mutex_lock(&dev_opp_list_lock); + mutex_lock(&opp_table_lock); - dev_opp = _find_device_opp(dev); - if (WARN_ON(IS_ERR(dev_opp))) { - ret = PTR_ERR(dev_opp); - mutex_unlock(&dev_opp_list_lock); + opp_table = _find_opp_table(dev); + if (WARN_ON(IS_ERR(opp_table))) { + ret = PTR_ERR(opp_table); + mutex_unlock(&opp_table_lock); goto free_table; } - dev_opp->np = opp_np; - dev_opp->shared_opp = of_property_read_bool(opp_np, "opp-shared"); + opp_table->np = opp_np; + opp_table->shared_opp = of_property_read_bool(opp_np, "opp-shared"); - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); return 0; @@ -1255,7 +1986,7 @@ */ nr = prop->length / sizeof(u32); if (nr % 2) { - dev_err(dev, "%s: Invalid OPP list\n", __func__); + dev_err(dev, "%s: Invalid OPP table\n", __func__); return -EINVAL; } @@ -1275,11 +2006,11 @@ /** * dev_pm_opp_of_add_table() - Initialize opp table from device tree - * @dev: device pointer used to lookup device OPPs. + * @dev: device pointer used to lookup OPP table. * * Register the initial OPP table with the OPP library for given device. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function indirectly uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where diff -ruw linux-4.4.115/drivers/base/power/opp/cpu.c linux-4.4.115-fbx/drivers/base/power/opp/cpu.c --- linux-4.4.115/drivers/base/power/opp/cpu.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/opp/cpu.c 2019-01-22 16:16:22.879240756 +0100 @@ -31,7 +31,7 @@ * @table: Cpufreq table returned back to caller * * Generate a cpufreq table for a provided device- this assumes that the - * opp list is already initialized and ready for usage. + * opp table is already initialized and ready for usage. * * This function allocates required memory for the cpufreq table. It is * expected that the caller does the required maintenance such as freeing @@ -44,7 +44,7 @@ * WARNING: It is important for the callers to ensure refreshing their copy of * the table if any of the mentioned functions have been invoked in the interim. * - * Locking: The internal device_opp and opp structures are RCU protected. + * Locking: The internal opp_table and opp structures are RCU protected. * Since we just use the regular accessor functions to access the internal data * structures, we use RCU read lock inside this function. As a result, users of * this function DONOT need to use explicit locks for invoking. @@ -122,15 +122,15 @@ /* Required only for V1 bindings, as v2 can manage it from DT itself */ int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask) { - struct device_list_opp *list_dev; - struct device_opp *dev_opp; + struct opp_device *opp_dev; + struct opp_table *opp_table; struct device *dev; int cpu, ret = 0; - mutex_lock(&dev_opp_list_lock); + mutex_lock(&opp_table_lock); - dev_opp = _find_device_opp(cpu_dev); - if (IS_ERR(dev_opp)) { + opp_table = _find_opp_table(cpu_dev); + if (IS_ERR(opp_table)) { ret = -EINVAL; goto unlock; } @@ -146,15 +146,15 @@ continue; } - list_dev = _add_list_dev(dev, dev_opp); - if (!list_dev) { - dev_err(dev, "%s: failed to add list-dev for cpu%d device\n", + opp_dev = _add_opp_dev(dev, opp_table); + if (!opp_dev) { + dev_err(dev, "%s: failed to add opp-dev for cpu%d device\n", __func__, cpu); continue; } } unlock: - mutex_unlock(&dev_opp_list_lock); + mutex_unlock(&opp_table_lock); return ret; } @@ -214,7 +214,6 @@ /* * Works only for OPP v2 bindings. * - * cpumask should be already set to mask of cpu_dev->id. * Returns -ENOENT if operating-points-v2 bindings aren't supported. */ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, cpumask_var_t cpumask) @@ -230,6 +229,8 @@ return -ENOENT; } + cpumask_set_cpu(cpu_dev->id, cpumask); + /* OPPs are shared ? */ if (!of_property_read_bool(np, "opp-shared")) goto put_cpu_node; diff -ruw linux-4.4.115/drivers/base/power/opp/Makefile linux-4.4.115-fbx/drivers/base/power/opp/Makefile --- linux-4.4.115/drivers/base/power/opp/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/opp/Makefile 2019-01-22 16:16:22.879240756 +0100 @@ -1,2 +1,3 @@ ccflags-$(CONFIG_DEBUG_DRIVER) := -DDEBUG obj-y += core.o cpu.o +obj-$(CONFIG_DEBUG_FS) += debugfs.o diff -ruw linux-4.4.115/drivers/base/power/opp/opp.h linux-4.4.115-fbx/drivers/base/power/opp/opp.h --- linux-4.4.115/drivers/base/power/opp/opp.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/opp/opp.h 2019-01-22 16:16:22.879240756 +0100 @@ -17,17 +17,21 @@ #include #include #include +#include #include #include #include +struct clk; +struct regulator; + /* Lock to allow exclusive modification to the device and opp lists */ -extern struct mutex dev_opp_list_lock; +extern struct mutex opp_table_lock; /* * Internal data structure organization with the OPP layer library is as * follows: - * dev_opp_list (root) + * opp_tables (root) * |- device 1 (represents voltage domain 1) * | |- opp 1 (availability, freq, voltage) * | |- opp 2 .. @@ -36,23 +40,24 @@ * |- device 2 (represents the next voltage domain) * ... * `- device m (represents mth voltage domain) - * device 1, 2.. are represented by dev_opp structure while each opp + * device 1, 2.. are represented by opp_table structure while each opp * is represented by the opp structure. */ /** * struct dev_pm_opp - Generic OPP description structure - * @node: opp list node. The nodes are maintained throughout the lifetime + * @node: opp table node. The nodes are maintained throughout the lifetime * of boot. It is expected only an optimal set of OPPs are * added to the library by the SoC framework. - * RCU usage: opp list is traversed with RCU locks. node + * RCU usage: opp table is traversed with RCU locks. node * modification is possible realtime, hence the modifications - * are protected by the dev_opp_list_lock for integrity. + * are protected by the opp_table_lock for integrity. * IMPORTANT: the opp nodes should be maintained in increasing * order. - * @dynamic: not-created from static DT entries. * @available: true/false - marks if this OPP as available or not + * @dynamic: not-created from static DT entries. * @turbo: true if turbo (boost) OPP + * @suspend: true if suspend OPP * @rate: Frequency in hertz * @u_volt: Target voltage in microvolts corresponding to this OPP * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP @@ -60,9 +65,10 @@ * @u_amp: Maximum current drawn by the device in microamperes * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's * frequency from any other OPP's frequency. - * @dev_opp: points back to the device_opp struct this opp belongs to + * @opp_table: points back to the opp_table struct this opp belongs to * @rcu_head: RCU callback head used for deferred freeing * @np: OPP's device node. + * @dentry: debugfs dentry pointer (per opp) * * This structure stores the OPP information for a given device. */ @@ -72,6 +78,7 @@ bool available; bool dynamic; bool turbo; + bool suspend; unsigned long rate; unsigned long u_volt; @@ -80,40 +87,60 @@ unsigned long u_amp; unsigned long clock_latency_ns; - struct device_opp *dev_opp; + struct opp_table *opp_table; struct rcu_head rcu_head; struct device_node *np; + +#ifdef CONFIG_DEBUG_FS + struct dentry *dentry; +#endif }; /** - * struct device_list_opp - devices managed by 'struct device_opp' + * struct opp_device - devices managed by 'struct opp_table' * @node: list node * @dev: device to which the struct object belongs * @rcu_head: RCU callback head used for deferred freeing + * @dentry: debugfs dentry pointer (per device) * - * This is an internal data structure maintaining the list of devices that are - * managed by 'struct device_opp'. + * This is an internal data structure maintaining the devices that are managed + * by 'struct opp_table'. */ -struct device_list_opp { +struct opp_device { struct list_head node; const struct device *dev; struct rcu_head rcu_head; + +#ifdef CONFIG_DEBUG_FS + struct dentry *dentry; +#endif }; /** - * struct device_opp - Device opp structure - * @node: list node - contains the devices with OPPs that + * struct opp_table - Device opp structure + * @node: table node - contains the devices with OPPs that * have been registered. Nodes once added are not modified in this - * list. - * RCU usage: nodes are not modified in the list of device_opp, - * however addition is possible and is secured by dev_opp_list_lock + * table. + * RCU usage: nodes are not modified in the table of opp_table, + * however addition is possible and is secured by opp_table_lock * @srcu_head: notifier head to notify the OPP availability changes. * @rcu_head: RCU callback head used for deferred freeing * @dev_list: list of devices that share these OPPs - * @opp_list: list of opps + * @opp_list: table of opps * @np: struct device_node pointer for opp's DT node. + * @clock_latency_ns_max: Max clock latency in nanoseconds. * @shared_opp: OPP is shared between multiple devices. + * @suspend_opp: Pointer to OPP to be used during device suspend. + * @supported_hw: Array of version number to support. + * @supported_hw_count: Number of elements in supported_hw array. + * @prop_name: A name to postfix to many DT properties, while parsing them. + * @clk: Device's clock handle + * @regulator: Supply regulator + * @dentry: debugfs dentry pointer of the real device directory (not links). + * @dentry_name: Name of the real dentry. + * + * @voltage_tolerance_v1: In percentage, for v1 bindings only. * * This is an internal data structure maintaining the link to opps attached to * a device. This structure is not meant to be shared to users as it is @@ -123,7 +150,7 @@ * need to wait for the grace period of both of them before freeing any * resources. And so we have used kfree_rcu() from within call_srcu() handlers. */ -struct device_opp { +struct opp_table { struct list_head node; struct srcu_notifier_head srcu_head; @@ -133,14 +160,48 @@ struct device_node *np; unsigned long clock_latency_ns_max; + + /* For backward compatibility with v1 bindings */ + unsigned int voltage_tolerance_v1; + bool shared_opp; struct dev_pm_opp *suspend_opp; + + unsigned int *supported_hw; + unsigned int supported_hw_count; + const char *prop_name; + struct clk *clk; + struct regulator *regulator; + +#ifdef CONFIG_DEBUG_FS + struct dentry *dentry; + char dentry_name[NAME_MAX]; +#endif }; /* Routines internal to opp core */ -struct device_opp *_find_device_opp(struct device *dev); -struct device_list_opp *_add_list_dev(const struct device *dev, - struct device_opp *dev_opp); +struct opp_table *_find_opp_table(struct device *dev); +struct opp_device *_add_opp_dev(const struct device *dev, struct opp_table *opp_table); struct device_node *_of_get_opp_desc_node(struct device *dev); +#ifdef CONFIG_DEBUG_FS +void opp_debug_remove_one(struct dev_pm_opp *opp); +int opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table); +int opp_debug_register(struct opp_device *opp_dev, struct opp_table *opp_table); +void opp_debug_unregister(struct opp_device *opp_dev, struct opp_table *opp_table); +#else +static inline void opp_debug_remove_one(struct dev_pm_opp *opp) {} + +static inline int opp_debug_create_one(struct dev_pm_opp *opp, + struct opp_table *opp_table) +{ return 0; } +static inline int opp_debug_register(struct opp_device *opp_dev, + struct opp_table *opp_table) +{ return 0; } + +static inline void opp_debug_unregister(struct opp_device *opp_dev, + struct opp_table *opp_table) +{ } +#endif /* DEBUG_FS */ + #endif /* __DRIVER_OPP_H__ */ diff -ruw linux-4.4.115/drivers/base/power/power.h linux-4.4.115-fbx/drivers/base/power/power.h --- linux-4.4.115/drivers/base/power/power.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/power.h 2019-01-22 16:16:22.879240756 +0100 @@ -140,6 +140,7 @@ extern void device_pm_move_before(struct device *, struct device *); extern void device_pm_move_after(struct device *, struct device *); extern void device_pm_move_last(struct device *); +extern void device_pm_check_callbacks(struct device *dev); #else /* !CONFIG_PM_SLEEP */ @@ -158,6 +159,8 @@ struct device *devb) {} static inline void device_pm_move_last(struct device *dev) {} +static inline void device_pm_check_callbacks(struct device *dev) {} + #endif /* !CONFIG_PM_SLEEP */ static inline void device_pm_init(struct device *dev) diff -ruw linux-4.4.115/drivers/base/power/qos.c linux-4.4.115-fbx/drivers/base/power/qos.c --- linux-4.4.115/drivers/base/power/qos.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/qos.c 2019-01-22 16:16:22.879240756 +0100 @@ -147,7 +147,7 @@ switch(req->type) { case DEV_PM_QOS_RESUME_LATENCY: ret = pm_qos_update_target(&qos->resume_latency, - &req->data.pnode, action, value); + &req->data.lat, action, value); if (ret) { value = pm_qos_read_value(&qos->resume_latency); blocking_notifier_call_chain(&dev_pm_notifiers, @@ -157,7 +157,7 @@ break; case DEV_PM_QOS_LATENCY_TOLERANCE: ret = pm_qos_update_target(&qos->latency_tolerance, - &req->data.pnode, action, value); + &req->data.lat, action, value); if (ret) { value = pm_qos_read_value(&qos->latency_tolerance); req->dev->power.set_latency_tolerance(req->dev, value); @@ -258,7 +258,7 @@ /* Flush the constraints lists for the device. */ c = &qos->resume_latency; - plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { + plist_for_each_entry_safe(req, tmp, &c->list, data.lat.node) { /* * Update constraints list and call the notification * callbacks if needed @@ -266,8 +266,11 @@ apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); memset(req, 0, sizeof(*req)); } + + kfree(c->notifiers); + c = &qos->latency_tolerance; - plist_for_each_entry_safe(req, tmp, &c->list, data.pnode) { + plist_for_each_entry_safe(req, tmp, &c->list, data.lat.node) { apply_constraint(req, PM_QOS_REMOVE_REQ, PM_QOS_DEFAULT_VALUE); memset(req, 0, sizeof(*req)); } @@ -382,7 +385,7 @@ switch(req->type) { case DEV_PM_QOS_RESUME_LATENCY: case DEV_PM_QOS_LATENCY_TOLERANCE: - curr_value = req->data.pnode.prio; + curr_value = req->data.lat.node.prio; break; case DEV_PM_QOS_FLAGS: curr_value = req->data.flr.flags; @@ -835,7 +838,7 @@ ret = IS_ERR_OR_NULL(dev->power.qos) || !dev->power.qos->latency_tolerance_req ? PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT : - dev->power.qos->latency_tolerance_req->data.pnode.prio; + dev->power.qos->latency_tolerance_req->data.lat.node.prio; mutex_unlock(&dev_pm_qos_mtx); return ret; } diff -ruw linux-4.4.115/drivers/base/power/wakeup.c linux-4.4.115-fbx/drivers/base/power/wakeup.c --- linux-4.4.115/drivers/base/power/wakeup.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/power/wakeup.c 2019-10-29 09:26:23.413200927 +0100 @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "power.h" @@ -808,6 +809,37 @@ } EXPORT_SYMBOL_GPL(pm_wakeup_event); +void pm_get_active_wakeup_sources(char *pending_wakeup_source, size_t max) +{ + struct wakeup_source *ws, *last_active_ws = NULL; + int len = 0; + bool active = false; + + rcu_read_lock(); + list_for_each_entry_rcu(ws, &wakeup_sources, entry) { + if (ws->active && len < max) { + if (!active) + len += scnprintf(pending_wakeup_source, max, + "Pending Wakeup Sources: "); + len += scnprintf(pending_wakeup_source + len, max - len, + "%s ", ws->name); + active = true; + } else if (!active && + (!last_active_ws || + ktime_to_ns(ws->last_time) > + ktime_to_ns(last_active_ws->last_time))) { + last_active_ws = ws; + } + } + if (!active && last_active_ws) { + scnprintf(pending_wakeup_source, max, + "Last active Wakeup Source: %s", + last_active_ws->name); + } + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(pm_get_active_wakeup_sources); + void pm_print_active_wakeup_sources(void) { struct wakeup_source *ws; @@ -1015,7 +1047,7 @@ active_time = ktime_set(0, 0); } - seq_printf(m, "%-12s\t%lu\t\t%lu\t\t%lu\t\t%lu\t\t%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", + seq_printf(m, "%-32s\t%lu\t\t%lu\t\t%lu\t\t%lu\t\t%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", ws->name, active_count, ws->event_count, ws->wakeup_count, ws->expire_count, ktime_to_ms(active_time), ktime_to_ms(total_time), @@ -1036,7 +1068,7 @@ struct wakeup_source *ws; int srcuidx; - seq_puts(m, "name\t\tactive_count\tevent_count\twakeup_count\t" + seq_puts(m, "name\t\t\t\t\tactive_count\tevent_count\twakeup_count\t" "expire_count\tactive_since\ttotal_time\tmax_time\t" "last_change\tprevent_suspend_time\n"); diff -ruw linux-4.4.115/drivers/base/regmap/internal.h linux-4.4.115-fbx/drivers/base/regmap/internal.h --- linux-4.4.115/drivers/base/regmap/internal.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/regmap/internal.h 2019-01-22 16:16:22.883240792 +0100 @@ -85,6 +85,9 @@ struct list_head debugfs_off_cache; struct mutex cache_lock; + + unsigned int dump_address; + unsigned int dump_count; #endif unsigned int max_register; @@ -245,6 +248,10 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, const void *val, size_t val_len); +int _regmap_raw_multi_reg_write(struct regmap *map, + const struct reg_sequence *regs, + size_t num_regs); + void regmap_async_complete_cb(struct regmap_async *async, int ret); enum regmap_endian regmap_get_val_endian(struct device *dev, diff -ruw linux-4.4.115/drivers/base/regmap/Kconfig linux-4.4.115-fbx/drivers/base/regmap/Kconfig --- linux-4.4.115/drivers/base/regmap/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/regmap/Kconfig 2019-01-22 16:16:22.883240792 +0100 @@ -3,7 +3,7 @@ # subsystems should select the appropriate symbols. config REGMAP - default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ) + default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ || REGMAP_SWR) select LZO_COMPRESS select LZO_DECOMPRESS select IRQ_DOMAIN if REGMAP_IRQ @@ -29,3 +29,15 @@ config REGMAP_IRQ bool + +config REGMAP_SWR + tristate + +config REGMAP_ALLOW_WRITE_DEBUGFS + depends on REGMAP && DEBUG_FS + bool "Allow REGMAP debugfs write" + default n + help + Say 'y' here to allow the regmap debugfs write. Regmap debugfs write + could be risky when accessing some essential hardwares, so it is not + recommended to enable this option on any production device. diff -ruw linux-4.4.115/drivers/base/regmap/Makefile linux-4.4.115-fbx/drivers/base/regmap/Makefile --- linux-4.4.115/drivers/base/regmap/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/regmap/Makefile 2019-01-22 16:16:22.883240792 +0100 @@ -10,3 +10,4 @@ obj-$(CONFIG_REGMAP_SPMI) += regmap-spmi.o obj-$(CONFIG_REGMAP_MMIO) += regmap-mmio.o obj-$(CONFIG_REGMAP_IRQ) += regmap-irq.o +obj-$(CONFIG_REGMAP_SWR) += regmap-swr.o diff -ruw linux-4.4.115/drivers/base/regmap/regcache.c linux-4.4.115-fbx/drivers/base/regmap/regcache.c --- linux-4.4.115/drivers/base/regmap/regcache.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/regmap/regcache.c 2019-01-22 16:16:22.883240792 +0100 @@ -682,6 +682,53 @@ return ret; } +static int regcache_sync_block_raw_multi_reg(struct regmap *map, void *block, + unsigned long *cache_present, + unsigned int block_base, + unsigned int start, + unsigned int end) +{ + unsigned int i, val; + unsigned int regtmp = 0; + int ret = 0; + struct reg_sequence *regs; + size_t num_regs = ((end - start) + 1); + + regs = kcalloc(num_regs, sizeof(struct reg_default), GFP_KERNEL); + if (!regs) + return -ENOMEM; + + num_regs = 0; + for (i = start; i < end; i++) { + regtmp = block_base + (i * map->reg_stride); + + /* skip registers that are not defined/available */ + if (!regcache_reg_present(cache_present, i)) + continue; + + val = regcache_get_val(map, block, i); + + /* Is this the hardware default? If so skip. */ + ret = regcache_lookup_reg(map, regtmp); + if (ret >= 0 && val == map->reg_defaults[ret].def) { + continue; + } else { + regs[num_regs].reg = regtmp; + regs[num_regs].def = val; + regs[num_regs].delay_us = 0; + num_regs += 1; + } + } + ret = 0; + if (num_regs) { + dev_dbg(map->dev, "%s: start: 0x%x - end: 0x%x\n", + __func__, regs[0].reg, regs[num_regs-1].reg); + ret = _regmap_raw_multi_reg_write(map, regs, num_regs); + } + kfree(regs); + return ret; +} + static int regcache_sync_block_raw(struct regmap *map, void *block, unsigned long *cache_present, unsigned int block_base, unsigned int start, @@ -729,7 +776,12 @@ unsigned int block_base, unsigned int start, unsigned int end) { - if (regmap_can_raw_write(map) && !map->use_single_write) + if (regmap_can_raw_write(map) && map->can_multi_write) + return regcache_sync_block_raw_multi_reg(map, block, + cache_present, + block_base, start, + end); + else if (regmap_can_raw_write(map) && !map->use_single_write) return regcache_sync_block_raw(map, block, cache_present, block_base, start, end); else diff -ruw linux-4.4.115/drivers/base/regmap/regmap.c linux-4.4.115-fbx/drivers/base/regmap/regmap.c --- linux-4.4.115/drivers/base/regmap/regmap.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/regmap/regmap.c 2019-10-29 09:26:23.413200927 +0100 @@ -1822,7 +1822,7 @@ * they are all in the same page and have been changed to being page * relative. The page register has been written if that was necessary. */ -static int _regmap_raw_multi_reg_write(struct regmap *map, +int _regmap_raw_multi_reg_write(struct regmap *map, const struct reg_sequence *regs, size_t num_regs) { diff -ruw linux-4.4.115/drivers/base/regmap/regmap-debugfs.c linux-4.4.115-fbx/drivers/base/regmap/regmap-debugfs.c --- linux-4.4.115/drivers/base/regmap/regmap-debugfs.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/regmap/regmap-debugfs.c 2019-01-22 16:16:22.883240792 +0100 @@ -259,8 +259,7 @@ count, ppos); } -#undef REGMAP_ALLOW_WRITE_DEBUGFS -#ifdef REGMAP_ALLOW_WRITE_DEBUGFS +#ifdef CONFIG_REGMAP_ALLOW_WRITE_DEBUGFS /* * This can be dangerous especially when we have clients such as * PMICs, therefore don't provide any real compile time configuration option @@ -310,6 +309,67 @@ .llseek = default_llseek, }; +static ssize_t regmap_data_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct regmap *map = file->private_data; + int new_count; + + regmap_calc_tot_len(map, NULL, 0); + new_count = map->dump_count * map->debugfs_tot_len; + if (new_count > count) + new_count = count; + + if (*ppos == 0) + *ppos = map->dump_address * map->debugfs_tot_len; + else if (*ppos >= map->dump_address * map->debugfs_tot_len + + map->dump_count * map->debugfs_tot_len) + return 0; + return regmap_read_debugfs(map, 0, map->max_register, user_buf, + new_count, ppos); +} + +#ifdef CONFIG_REGMAP_ALLOW_WRITE_DEBUGFS +static ssize_t regmap_data_write_file(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[32]; + size_t buf_size; + char *start = buf; + unsigned long value; + struct regmap *map = file->private_data; + int ret; + + buf_size = min(count, (sizeof(buf)-1)); + if (copy_from_user(buf, user_buf, buf_size)) + return -EFAULT; + buf[buf_size] = 0; + + while (*start == ' ') + start++; + if (kstrtoul(start, 16, &value)) + return -EINVAL; + + /* Userspace has been fiddling around behind the kernel's back */ + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + + ret = regmap_write(map, map->dump_address, value); + if (ret < 0) + return ret; + return buf_size; +} +#else +#define regmap_data_write_file NULL +#endif + +static const struct file_operations regmap_data_fops = { + .open = simple_open, + .read = regmap_data_read_file, + .write = regmap_data_write_file, + .llseek = default_llseek, +}; + static ssize_t regmap_range_read_file(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { @@ -595,7 +655,7 @@ if (map->max_register || regmap_readable(map, 0)) { umode_t registers_mode; -#if defined(REGMAP_ALLOW_WRITE_DEBUGFS) +#ifdef CONFIG_REGMAP_ALLOW_WRITE_DEBUGFS registers_mode = 0600; #else registers_mode = 0400; @@ -603,6 +663,15 @@ debugfs_create_file("registers", registers_mode, map->debugfs, map, ®map_map_fops); + + debugfs_create_x32("address", 0600, map->debugfs, + &map->dump_address); + map->dump_count = 1; + debugfs_create_u32("count", 0600, map->debugfs, + &map->dump_count); + debugfs_create_file("data", registers_mode, map->debugfs, + map, ®map_data_fops); + debugfs_create_file("access", 0400, map->debugfs, map, ®map_access_fops); } diff -ruw linux-4.4.115/drivers/base/regmap/regmap-spmi.c linux-4.4.115-fbx/drivers/base/regmap/regmap-spmi.c --- linux-4.4.115/drivers/base/regmap/regmap-spmi.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/regmap/regmap-spmi.c 2019-01-22 16:16:22.883240792 +0100 @@ -1,7 +1,7 @@ /* * Register map access API - SPMI support * - * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. + * Copyright (c) 2012-2013,2016 The Linux Foundation. All rights reserved. * * Based on regmap-i2c.c: * Copyright 2011 Wolfson Microelectronics plc diff -ruw linux-4.4.115/drivers/base/syscore.c linux-4.4.115-fbx/drivers/base/syscore.c --- linux-4.4.115/drivers/base/syscore.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/base/syscore.c 2019-01-22 16:16:22.883240792 +0100 @@ -11,6 +11,7 @@ #include #include #include +#include static LIST_HEAD(syscore_ops_list); static DEFINE_MUTEX(syscore_ops_lock); @@ -75,6 +76,8 @@ return 0; err_out: + log_suspend_abort_reason("System core suspend callback %pF failed", + ops->suspend); pr_err("PM: System core suspend callback %pF failed.\n", ops->suspend); list_for_each_entry_continue(ops, &syscore_ops_list, node) diff -ruw linux-4.4.115/drivers/bluetooth/hci_h4.c linux-4.4.115-fbx/drivers/bluetooth/hci_h4.c --- linux-4.4.115/drivers/bluetooth/hci_h4.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/bluetooth/hci_h4.c 2019-01-22 16:16:22.935241263 +0100 @@ -57,7 +57,7 @@ { struct h4_struct *h4; - BT_DBG("hu %p", hu); + BT_DBG("hu %pK", hu); h4 = kzalloc(sizeof(*h4), GFP_KERNEL); if (!h4) @@ -74,7 +74,7 @@ { struct h4_struct *h4 = hu->priv; - BT_DBG("hu %p", hu); + BT_DBG("hu %pK", hu); skb_queue_purge(&h4->txq); @@ -88,7 +88,7 @@ hu->priv = NULL; - BT_DBG("hu %p", hu); + BT_DBG("hu %pK", hu); skb_queue_purge(&h4->txq); @@ -105,7 +105,7 @@ { struct h4_struct *h4 = hu->priv; - BT_DBG("hu %p skb %p", hu, skb); + BT_DBG("hu %pK skb %pK", hu, skb); /* Prepend skb with frame type */ memcpy(skb_push(skb, 1), &bt_cb(skb)->pkt_type, 1); diff -ruw linux-4.4.115/drivers/bluetooth/hci_ldisc.c linux-4.4.115-fbx/drivers/bluetooth/hci_ldisc.c --- linux-4.4.115/drivers/bluetooth/hci_ldisc.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/bluetooth/hci_ldisc.c 2019-10-29 09:26:23.437201162 +0100 @@ -142,6 +142,8 @@ struct hci_dev *hdev = hu->hdev; struct sk_buff *skb; + BT_DBG("hu %pK hdev %pK tty %pK", hu, hdev, tty); + /* REVISIT: should we cope with bad skbs or ->write() returning * and error value ? */ @@ -205,7 +207,7 @@ /* Initialize device */ static int hci_uart_open(struct hci_dev *hdev) { - BT_DBG("%s %p", hdev->name, hdev); + BT_DBG("%s %pK", hdev->name, hdev); /* Nothing to do for UART driver */ return 0; @@ -217,7 +219,7 @@ struct hci_uart *hu = hci_get_drvdata(hdev); struct tty_struct *tty = hu->tty; - BT_DBG("hdev %p tty %p", hdev, tty); + BT_DBG("hdev %pK tty %pK", hdev, tty); if (hu->tx_skb) { kfree_skb(hu->tx_skb); hu->tx_skb = NULL; @@ -236,7 +238,7 @@ /* Close device */ static int hci_uart_close(struct hci_dev *hdev) { - BT_DBG("hdev %p", hdev); + BT_DBG("hdev %pK", hdev); hci_uart_flush(hdev); hdev->flush = NULL; @@ -441,7 +443,7 @@ { struct hci_uart *hu; - BT_DBG("tty %p", tty); + BT_DBG("tty %pK", tty); /* Error if the tty has no write op instead of leaving an exploitable hole */ @@ -483,7 +485,7 @@ struct hci_uart *hu = tty->disc_data; struct hci_dev *hdev; - BT_DBG("tty %p", tty); + BT_DBG("tty %pK", tty); /* Detach from the tty */ tty->disc_data = NULL; diff -ruw linux-4.4.115/drivers/bluetooth/Kconfig linux-4.4.115-fbx/drivers/bluetooth/Kconfig --- linux-4.4.115/drivers/bluetooth/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/bluetooth/Kconfig 2019-01-22 16:16:22.927241190 +0100 @@ -320,4 +320,32 @@ Say Y here to compile support for Texas Instrument's WiLink7 driver into the kernel or say M to compile it as module (btwilink). +config MSM_BT_POWER + tristate "MSM Bluetooth Power Control" + depends on ARCH_QCOM && RFKILL + default m + help + Provides a parameter to switch on/off power from PMIC + to Bluetooth device. + +config BTFM_SLIM + tristate "MSM Bluetooth/FM Slimbus Driver" + depends on MSM_BT_POWER + help + This enables BT/FM slimbus driver to get multiple audio channel. + This will make use of slimbus platform driver and slimbus codec + driver to communicate with slimbus machine driver and LPSS which + is Slimbus master. + + Slimbus slave initialization and configuration will be done through + this driver. + +config BTFM_SLIM_WCN3990 + tristate "MSM Bluetooth/FM WCN3990 Device" + depends on BTFM_SLIM + help + This enables specific driver handle for WCN3990 device. + It is designed to adapt any future BT/FM device to implement a specific + chip initialization process and control. + endmenu diff -ruw linux-4.4.115/drivers/bluetooth/Makefile linux-4.4.115-fbx/drivers/bluetooth/Makefile --- linux-4.4.115/drivers/bluetooth/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/bluetooth/Makefile 2019-01-22 16:16:22.927241190 +0100 @@ -23,6 +23,11 @@ obj-$(CONFIG_BT_BCM) += btbcm.o obj-$(CONFIG_BT_RTL) += btrtl.o obj-$(CONFIG_BT_QCA) += btqca.o +obj-$(CONFIG_MSM_BT_POWER) += bluetooth-power.o + +obj-$(CONFIG_BTFM_SLIM) += btfm_slim.o +obj-$(CONFIG_BTFM_SLIM) += btfm_slim_codec.o +obj-$(CONFIG_BTFM_SLIM_WCN3990) += btfm_slim_wcn3990.o btmrvl-y := btmrvl_main.o btmrvl-$(CONFIG_DEBUG_FS) += btmrvl_debugfs.o diff -ruw linux-4.4.115/drivers/char/hw_random/Kconfig linux-4.4.115-fbx/drivers/char/hw_random/Kconfig --- linux-4.4.115/drivers/char/hw_random/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/char/hw_random/Kconfig 2019-01-22 16:16:22.967241553 +0100 @@ -333,10 +333,25 @@ If unsure, say Y. +config HW_RANDOM_MSM_LEGACY + tristate "Qualcomm MSM Random Number Generator support (LEGACY)" + depends on HW_RANDOM && ARCH_QCOM + select CRYPTO_AES + select CRYPTO_ECB + default n + ---help--- + This driver provides kernel-side support for the Random Number + Generator hardware found on Qualcomm MSM SoCs. + + To compile this driver as a module, choose M here: the + module will be called msm_rng. + + If unsure, say Y. + config HW_RANDOM_MSM tristate "Qualcomm SoCs Random Number Generator support" depends on HW_RANDOM && ARCH_QCOM - default HW_RANDOM + default n ---help--- This driver provides kernel-side support for the Random Number Generator hardware found on Qualcomm SoCs. diff -ruw linux-4.4.115/drivers/char/hw_random/Makefile linux-4.4.115-fbx/drivers/char/hw_random/Makefile --- linux-4.4.115/drivers/char/hw_random/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/char/hw_random/Makefile 2019-01-22 16:16:22.967241553 +0100 @@ -32,4 +32,5 @@ obj-$(CONFIG_HW_RANDOM_MSM) += msm-rng.o obj-$(CONFIG_HW_RANDOM_ST) += st-rng.o obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o +obj-$(CONFIG_HW_RANDOM_MSM_LEGACY) += msm_rng.o obj-$(CONFIG_HW_RANDOM_STM32) += stm32-rng.o diff -ruw linux-4.4.115/drivers/char/Kconfig linux-4.4.115-fbx/drivers/char/Kconfig --- linux-4.4.115/drivers/char/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/char/Kconfig 2019-10-29 09:26:23.441201201 +0100 @@ -48,6 +48,8 @@ source "drivers/tty/serial/Kconfig" +source "drivers/char/diag/Kconfig" + config TTY_PRINTK tristate "TTY driver to output user messages via printk" depends on EXPERT && TTY @@ -592,6 +594,16 @@ source "drivers/s390/char/Kconfig" +config MSM_SMD_PKT + bool "Enable device interface for some SMD packet ports" + default n + depends on MSM_SMD + help + smd_pkt driver provides the interface for the userspace clients + to communicate over smd via device nodes. This enable the + usersapce clients to read and write to some smd packets channel + for MSM chipset. + config TILE_SROM bool "Character-device access via hypervisor to the Tilera SPI ROM" depends on TILE @@ -605,5 +617,21 @@ source "drivers/char/xillybus/Kconfig" +config MSM_ADSPRPC + tristate "QTI ADSP RPC driver" + depends on MSM_SMD + help + Provides a communication mechanism that allows for clients to + make remote method invocations across processor boundary to + applications DSP processor. Say M if you want to enable this + module. + +config MSM_RDBG + tristate "QTI Remote debug driver" + help + Implements a shared memory based transport mechanism that allows + for a debugger running on a host PC to communicate with a remote + stub running on peripheral subsystems such as the ADSP, MODEM etc. + endmenu diff -ruw linux-4.4.115/drivers/char/Makefile linux-4.4.115-fbx/drivers/char/Makefile --- linux-4.4.115/drivers/char/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/char/Makefile 2019-10-29 09:26:23.441201201 +0100 @@ -9,6 +9,7 @@ obj-$(CONFIG_VIRTIO_CONSOLE) += virtio_console.o obj-$(CONFIG_RAW_DRIVER) += raw.o obj-$(CONFIG_SGI_SNSC) += snsc.o snsc_event.o +obj-$(CONFIG_MSM_SMD_PKT) += msm_smd_pkt.o obj-$(CONFIG_MSPEC) += mspec.o obj-$(CONFIG_MMTIMER) += mmtimer.o obj-$(CONFIG_UV_MMTIMER) += uv_mmtimer.o @@ -59,4 +60,10 @@ js-rtc-y = rtc.o obj-$(CONFIG_TILE_SROM) += tile-srom.o +obj-$(CONFIG_DIAG_CHAR) += diag/ obj-$(CONFIG_XILLYBUS) += xillybus/ +obj-$(CONFIG_MSM_ADSPRPC) += adsprpc.o +ifdef CONFIG_COMPAT +obj-$(CONFIG_MSM_ADSPRPC) += adsprpc_compat.o +endif +obj-$(CONFIG_MSM_RDBG) += rdbg.o diff -ruw linux-4.4.115/drivers/char/misc.c linux-4.4.115-fbx/drivers/char/misc.c --- linux-4.4.115/drivers/char/misc.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/char/misc.c 2019-01-22 16:16:22.975241625 +0100 @@ -59,7 +59,7 @@ /* * Assigned numbers, used for dynamic minors */ -#define DYNAMIC_MINORS 64 /* like dynamic majors */ +#define DYNAMIC_MINORS 96 /* like dynamic majors */ static DECLARE_BITMAP(misc_minors, DYNAMIC_MINORS); #ifdef CONFIG_PROC_FS diff -ruw linux-4.4.115/drivers/clk/clk.c linux-4.4.115-fbx/drivers/clk/clk.c --- linux-4.4.115/drivers/clk/clk.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clk/clk.c 2019-10-29 09:26:23.469201475 +0100 @@ -1,6 +1,7 @@ /* * Copyright (C) 2010-2011 Canonical Ltd * Copyright (C) 2011-2012 Linaro Ltd + * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -23,9 +24,12 @@ #include #include #include +#include #include "clk.h" +#if defined(CONFIG_COMMON_CLK) + static DEFINE_SPINLOCK(enable_lock); static DEFINE_MUTEX(prepare_lock); @@ -39,6 +43,13 @@ static HLIST_HEAD(clk_orphan_list); static LIST_HEAD(clk_notifier_list); +struct clk_handoff_vdd { + struct list_head list; + struct clk_vdd_class *vdd_class; +}; + +static LIST_HEAD(clk_handoff_vdd_list); + /*** private data structures ***/ struct clk_core { @@ -60,6 +71,8 @@ bool orphan; unsigned int enable_count; unsigned int prepare_count; + bool need_handoff_enable; + bool need_handoff_prepare; unsigned long min_rate; unsigned long max_rate; unsigned long accuracy; @@ -73,6 +86,9 @@ struct hlist_node debug_node; #endif struct kref ref; + struct clk_vdd_class *vdd_class; + unsigned long *rate_max; + int num_rate_max; }; #define CREATE_TRACE_POINTS @@ -172,6 +188,9 @@ return core->ops->is_enabled(core->hw); } +static void clk_core_unprepare(struct clk_core *core); +static void clk_core_disable(struct clk_core *core); + static void clk_unprepare_unused_subtree(struct clk_core *core) { struct clk_core *child; @@ -181,6 +200,19 @@ hlist_for_each_entry(child, &core->children, child_node) clk_unprepare_unused_subtree(child); + /* + * setting CLK_ENABLE_HAND_OFF flag triggers this conditional + * + * need_handoff_prepare implies this clk was already prepared by + * __clk_init. now we have a proper user, so unset the flag in our + * internal bookkeeping. See CLK_ENABLE_HAND_OFF flag in clk-provider.h + * for details. + */ + if (core->need_handoff_prepare) { + core->need_handoff_prepare = false; + clk_core_unprepare(core); + } + if (core->prepare_count) return; @@ -207,6 +239,21 @@ hlist_for_each_entry(child, &core->children, child_node) clk_disable_unused_subtree(child); + /* + * setting CLK_ENABLE_HAND_OFF flag triggers this conditional + * + * need_handoff_enable implies this clk was already enabled by + * __clk_init. now we have a proper user, so unset the flag in our + * internal bookkeeping. See CLK_ENABLE_HAND_OFF flag in clk-provider.h + * for details. + */ + if (core->need_handoff_enable) { + core->need_handoff_enable = false; + flags = clk_enable_lock(); + clk_core_disable(core); + clk_enable_unlock(flags); + } + flags = clk_enable_lock(); if (core->enable_count) @@ -241,9 +288,12 @@ } __setup("clk_ignore_unused", clk_ignore_unused_setup); +static int clk_unvote_vdd_level(struct clk_vdd_class *vdd_class, int level); + static int clk_disable_unused(void) { struct clk_core *core; + struct clk_handoff_vdd *v, *v_temp; if (clk_ignore_unused) { pr_warn("clk: Not disabling unused clocks\n"); @@ -264,6 +314,13 @@ hlist_for_each_entry(core, &clk_orphan_list, child_node) clk_unprepare_unused_subtree(core); + list_for_each_entry_safe(v, v_temp, &clk_handoff_vdd_list, list) { + clk_unvote_vdd_level(v->vdd_class, + v->vdd_class->num_levels - 1); + list_del(&v->list); + kfree(v); + }; + clk_prepare_unlock(); return 0; @@ -545,6 +602,26 @@ EXPORT_SYMBOL_GPL(clk_hw_set_rate_range); /* + * Aggregate the rate of all child nodes which are enabled and exclude the + * child node which requests for clk_aggregate_rate. + */ +unsigned long clk_aggregate_rate(struct clk_hw *hw, + const struct clk_core *parent) +{ + struct clk_core *child; + unsigned long aggre_rate = 0; + + hlist_for_each_entry(child, &parent->children, child_node) { + if (child->enable_count && + strcmp(child->name, hw->init->name)) + aggre_rate = max(child->rate, aggre_rate); + } + + return aggre_rate; +} +EXPORT_SYMBOL_GPL(clk_aggregate_rate); + +/* * Helper for finding best parent to provide a given frequency. This can be used * directly as a determine_rate callback (e.g. for a mux), or from a more * complex clock that may combine a mux with other operations. @@ -563,6 +640,217 @@ } EXPORT_SYMBOL_GPL(__clk_mux_determine_rate_closest); +/* + * Find the voltage level required for a given clock rate. + */ +static int clk_find_vdd_level(struct clk_core *clk, unsigned long rate) +{ + int level; + + for (level = 0; level < clk->num_rate_max; level++) + if (rate <= clk->rate_max[level]) + break; + + if (level == clk->num_rate_max) { + pr_err("Rate %lu for %s is greater than highest Fmax\n", rate, + clk->name); + return -EINVAL; + } + + return level; +} + +/* + * Update voltage level given the current votes. + */ +static int clk_update_vdd(struct clk_vdd_class *vdd_class) +{ + int level, rc = 0, i, ignore; + struct regulator **r = vdd_class->regulator; + int *uv = vdd_class->vdd_uv; + int n_reg = vdd_class->num_regulators; + int cur_lvl = vdd_class->cur_level; + int max_lvl = vdd_class->num_levels - 1; + int cur_base = cur_lvl * n_reg; + int new_base; + + /* aggregate votes */ + for (level = max_lvl; level > 0; level--) + if (vdd_class->level_votes[level]) + break; + + if (level == cur_lvl) + return 0; + + max_lvl = max_lvl * n_reg; + new_base = level * n_reg; + + for (i = 0; i < vdd_class->num_regulators; i++) { + pr_debug("Set Voltage level Min %d, Max %d\n", uv[new_base + i], + uv[max_lvl + i]); + rc = regulator_set_voltage(r[i], uv[new_base + i], + vdd_class->use_max_uV ? INT_MAX : uv[max_lvl + i]); + if (rc) + goto set_voltage_fail; + + if (cur_lvl == 0 || cur_lvl == vdd_class->num_levels) + rc = regulator_enable(r[i]); + else if (level == 0) + rc = regulator_disable(r[i]); + if (rc) + goto enable_disable_fail; + } + + if (vdd_class->set_vdd && !vdd_class->num_regulators) + rc = vdd_class->set_vdd(vdd_class, level); + + if (!rc) + vdd_class->cur_level = level; + + return rc; + +enable_disable_fail: + regulator_set_voltage(r[i], uv[cur_base + i], + vdd_class->use_max_uV ? INT_MAX : uv[max_lvl + i]); + +set_voltage_fail: + for (i--; i >= 0; i--) { + regulator_set_voltage(r[i], uv[cur_base + i], + vdd_class->use_max_uV ? INT_MAX : uv[max_lvl + i]); + if (cur_lvl == 0 || cur_lvl == vdd_class->num_levels) + regulator_disable(r[i]); + else if (level == 0) + ignore = regulator_enable(r[i]); + } + + return rc; +} + +/* + * Vote for a voltage level. + */ +static int clk_vote_vdd_level(struct clk_vdd_class *vdd_class, int level) +{ + int rc = 0; + + if (level >= vdd_class->num_levels) + return -EINVAL; + + mutex_lock(&vdd_class->lock); + + vdd_class->level_votes[level]++; + + rc = clk_update_vdd(vdd_class); + if (rc) + vdd_class->level_votes[level]--; + + mutex_unlock(&vdd_class->lock); + + return rc; +} + +/* + * Remove vote for a voltage level. + */ +static int clk_unvote_vdd_level(struct clk_vdd_class *vdd_class, int level) +{ + int rc = 0; + + if (level >= vdd_class->num_levels) + return -EINVAL; + + mutex_lock(&vdd_class->lock); + + if (WARN(!vdd_class->level_votes[level], + "Reference counts are incorrect for %s level %d\n", + vdd_class->class_name, level)) + goto out; + + vdd_class->level_votes[level]--; + + rc = clk_update_vdd(vdd_class); + if (rc) + vdd_class->level_votes[level]++; + +out: + mutex_unlock(&vdd_class->lock); + return rc; +} + +/* + * Vote for a voltage level corresponding to a clock's rate. + */ +static int clk_vote_rate_vdd(struct clk_core *core, unsigned long rate) +{ + int level; + + if (!core->vdd_class) + return 0; + + level = clk_find_vdd_level(core, rate); + if (level < 0) + return level; + + return clk_vote_vdd_level(core->vdd_class, level); +} + +/* + * Remove vote for a voltage level corresponding to a clock's rate. + */ +static void clk_unvote_rate_vdd(struct clk_core *core, unsigned long rate) +{ + int level; + + if (!core->vdd_class) + return; + + level = clk_find_vdd_level(core, rate); + if (level < 0) + return; + + clk_unvote_vdd_level(core->vdd_class, level); +} + +static bool clk_is_rate_level_valid(struct clk_core *core, unsigned long rate) +{ + int level; + + if (!core->vdd_class) + return true; + + level = clk_find_vdd_level(core, rate); + + return level >= 0; +} + +static int clk_vdd_class_init(struct clk_vdd_class *vdd) +{ + struct clk_handoff_vdd *v; + + if (vdd->skip_handoff) + return 0; + + list_for_each_entry(v, &clk_handoff_vdd_list, list) { + if (v->vdd_class == vdd) + return 0; + } + + pr_debug("voting for vdd_class %s\n", vdd->class_name); + + if (clk_vote_vdd_level(vdd, vdd->num_levels - 1)) + pr_err("failed to vote for %s\n", vdd->class_name); + + v = kmalloc(sizeof(*v), GFP_KERNEL); + if (!v) + return -ENOMEM; + + v->vdd_class = vdd; + + list_add_tail(&v->list, &clk_handoff_vdd_list); + + return 0; +} + /*** clk api ***/ static void clk_core_unprepare(struct clk_core *core) @@ -575,6 +863,9 @@ if (WARN_ON(core->prepare_count == 0)) return; + if (WARN_ON(core->prepare_count == 1 && core->flags & CLK_IS_CRITICAL)) + return; + if (--core->prepare_count > 0) return; @@ -586,6 +877,9 @@ core->ops->unprepare(core->hw); trace_clk_unprepare_complete(core); + + clk_unvote_rate_vdd(core, core->rate); + clk_core_unprepare(core->parent); } @@ -627,12 +921,19 @@ trace_clk_prepare(core); + ret = clk_vote_rate_vdd(core, core->rate); + if (ret) { + clk_core_unprepare(core->parent); + return ret; + } + if (core->ops->prepare) ret = core->ops->prepare(core->hw); trace_clk_prepare_complete(core); if (ret) { + clk_unvote_rate_vdd(core, core->rate); clk_core_unprepare(core->parent); return ret; } @@ -657,7 +958,7 @@ */ int clk_prepare(struct clk *clk) { - int ret; + int ret = 0; if (!clk) return 0; @@ -680,6 +981,9 @@ if (WARN_ON(core->enable_count == 0)) return; + if (WARN_ON(core->enable_count == 1 && core->flags & CLK_IS_CRITICAL)) + return; + if (--core->enable_count > 0) return; @@ -769,7 +1073,7 @@ int clk_enable(struct clk *clk) { unsigned long flags; - int ret; + int ret = 0; if (!clk) return 0; @@ -1379,6 +1683,9 @@ top = clk_calc_new_rates(parent, best_parent_rate); out: + if (!clk_is_rate_level_valid(core, rate)) + return NULL; + clk_calc_subtree(core, new_rate, parent, p_index); return top; @@ -1427,7 +1734,7 @@ * walk down a subtree and set the new rates notifying the rate * change on the way */ -static void clk_change_rate(struct clk_core *core) +static int clk_change_rate(struct clk_core *core) { struct clk_core *child; struct hlist_node *tmp; @@ -1435,6 +1742,7 @@ unsigned long best_parent_rate = 0; bool skip_set_rate = false; struct clk_core *old_parent; + int rc = 0; old_rate = core->rate; @@ -1443,6 +1751,15 @@ else if (core->parent) best_parent_rate = core->parent->rate; + trace_clk_set_rate(core, core->new_rate); + + /* Enforce vdd requirements for new frequency. */ + if (core->prepare_count) { + rc = clk_vote_rate_vdd(core, core->new_rate); + if (rc) + goto out; + } + if (core->new_parent && core->new_parent != core->parent) { old_parent = __clk_set_parent_before(core, core->new_parent); trace_clk_set_parent(core, core->new_parent); @@ -1460,13 +1777,19 @@ __clk_set_parent_after(core, core->new_parent, old_parent); } - trace_clk_set_rate(core, core->new_rate); - - if (!skip_set_rate && core->ops->set_rate) - core->ops->set_rate(core->hw, core->new_rate, best_parent_rate); + if (!skip_set_rate && core->ops->set_rate) { + rc = core->ops->set_rate(core->hw, core->new_rate, + best_parent_rate); + if (rc) + goto err_set_rate; + } trace_clk_set_rate_complete(core, core->new_rate); + /* Release vdd requirements for old frequency. */ + if (core->prepare_count) + clk_unvote_rate_vdd(core, old_rate); + core->rate = clk_recalc(core, best_parent_rate); if (core->notifier_count && old_rate != core->rate) @@ -1489,6 +1812,16 @@ /* handle the new child who might not be in core->children yet */ if (core->new_child) clk_change_rate(core->new_child); + + return rc; + +err_set_rate: + if (core->prepare_count) + clk_unvote_rate_vdd(core, core->new_rate); +out: + trace_clk_set_rate_complete(core, core->new_rate); + + return rc; } static int clk_core_set_rate_nolock(struct clk_core *core, @@ -1523,7 +1856,13 @@ } /* change the rates */ - clk_change_rate(top); + ret = clk_change_rate(top); + if (ret) { + pr_err("%s: failed to set %s rate\n", __func__, + top->name); + clk_propagate_rate_change(top, ABORT_RATE_CHANGE); + return ret; + } core->req_rate = req_rate; @@ -1953,6 +2292,18 @@ } EXPORT_SYMBOL_GPL(clk_is_match); +int clk_set_flags(struct clk *clk, unsigned long flags) +{ + if (!clk) + return 0; + + if (!clk->core->ops->set_flags) + return -EINVAL; + + return clk->core->ops->set_flags(clk->core->hw, flags); +} +EXPORT_SYMBOL_GPL(clk_set_flags); + /*** debugfs support ***/ #ifdef CONFIG_DEBUG_FS @@ -1960,6 +2311,7 @@ static struct dentry *rootdir; static int inited = 0; +static u32 debug_suspend; static DEFINE_MUTEX(clk_debug_lock); static HLIST_HEAD(clk_debug_list); @@ -1974,6 +2326,56 @@ NULL, }; +static void clk_state_subtree(struct clk_core *c) +{ + int vdd_level = 0; + struct clk_core *child; + + if (!c) + return; + + if (c->vdd_class) { + vdd_level = clk_find_vdd_level(c, c->rate); + if (vdd_level < 0) + vdd_level = 0; + } + + trace_clk_state(c->name, c->prepare_count, c->enable_count, + c->rate, vdd_level); + + hlist_for_each_entry(child, &c->children, child_node) + clk_state_subtree(child); +} + +static int clk_state_show(struct seq_file *s, void *data) +{ + struct clk_core *c; + struct hlist_head **lists = (struct hlist_head **)s->private; + + clk_prepare_lock(); + + for (; *lists; lists++) + hlist_for_each_entry(c, *lists, child_node) + clk_state_subtree(c); + + clk_prepare_unlock(); + + return 0; +} + + +static int clk_state_open(struct inode *inode, struct file *file) +{ + return single_open(file, clk_state_show, inode->i_private); +} + +static const struct file_operations clk_state_fops = { + .open = clk_state_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + static void clk_summary_show_one(struct seq_file *s, struct clk_core *c, int level) { @@ -2102,6 +2504,349 @@ .release = single_release, }; +static int clock_debug_rate_set(void *data, u64 val) +{ + struct clk_core *core = data; + int ret; + + ret = clk_set_rate(core->hw->clk, val); + if (ret) + pr_err("clk_set_rate(%lu) failed (%d)\n", + (unsigned long)val, ret); + + return ret; +} + +static int clock_debug_rate_get(void *data, u64 *val) +{ + struct clk_core *core = data; + + *val = core->hw->core->rate; + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(clock_rate_fops, clock_debug_rate_get, + clock_debug_rate_set, "%llu\n"); + +static ssize_t clock_parent_read(struct file *filp, char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + char name[256] = {0}; + struct clk_core *core = filp->private_data; + struct clk_core *p = core->hw->core->parent; + + snprintf(name, sizeof(name), "%s\n", p ? p->name : "None\n"); + + return simple_read_from_buffer(ubuf, cnt, ppos, name, strlen(name)); +} + +static const struct file_operations clock_parent_fops = { + .open = simple_open, + .read = clock_parent_read, +}; + +static int clock_debug_enable_set(void *data, u64 val) +{ + struct clk_core *core = data; + int rc = 0; + + if (val) + rc = clk_prepare_enable(core->hw->clk); + else + clk_disable_unprepare(core->hw->clk); + + return rc; +} + +static int clock_debug_enable_get(void *data, u64 *val) +{ + struct clk_core *core = data; + int enabled = 0; + + enabled = core->enable_count; + + *val = enabled; + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(clock_enable_fops, clock_debug_enable_get, + clock_debug_enable_set, "%lld\n"); + +#define clock_debug_output(m, c, fmt, ...) \ +do { \ + if (m) \ + seq_printf(m, fmt, ##__VA_ARGS__); \ + else if (c) \ + pr_cont(fmt, ##__VA_ARGS__); \ + else \ + pr_info(fmt, ##__VA_ARGS__); \ +} while (0) + +/* + * clock_debug_print_enabled_debug_suspend() - Print names of enabled clocks + * during suspend. + */ +static void clock_debug_print_enabled_debug_suspend(struct seq_file *s) +{ + struct clk_core *core; + int cnt = 0; + + if (!mutex_trylock(&clk_debug_lock)) + return; + + clock_debug_output(s, 0, "Enabled clocks:\n"); + + hlist_for_each_entry(core, &clk_debug_list, debug_node) { + if (!core || !core->prepare_count) + continue; + + if (core->vdd_class) + clock_debug_output(s, 0, " %s:%u:%u [%ld, %d]", + core->name, core->prepare_count, + core->enable_count, core->rate, + clk_find_vdd_level(core, core->rate)); + + else + clock_debug_output(s, 0, " %s:%u:%u [%ld]", + core->name, core->prepare_count, + core->enable_count, core->rate); + cnt++; + } + + mutex_unlock(&clk_debug_lock); + + if (cnt) + clock_debug_output(s, 0, "Enabled clock count: %d\n", cnt); + else + clock_debug_output(s, 0, "No clocks enabled.\n"); +} + +static int clock_debug_print_clock(struct clk_core *c, struct seq_file *s) +{ + char *start = ""; + struct clk *clk; + + if (!c || !c->prepare_count) + return 0; + + clk = c->hw->clk; + + clock_debug_output(s, 0, "\t"); + + do { + if (clk->core->vdd_class) + clock_debug_output(s, 1, "%s%s:%u:%u [%ld, %d]", start, + clk->core->name, + clk->core->prepare_count, + clk->core->enable_count, + clk->core->rate, + clk_find_vdd_level(clk->core, clk->core->rate)); + else + clock_debug_output(s, 1, "%s%s:%u:%u [%ld]", start, + clk->core->name, + clk->core->prepare_count, + clk->core->enable_count, + clk->core->rate); + start = " -> "; + } while ((clk = clk_get_parent(clk))); + + clock_debug_output(s, 1, "\n"); + + return 1; +} + +/* + * clock_debug_print_enabled_clocks() - Print names of enabled clocks + */ +static void clock_debug_print_enabled_clocks(struct seq_file *s) +{ + struct clk_core *core; + int cnt = 0; + + clock_debug_output(s, 0, "Enabled clocks:\n"); + + mutex_lock(&clk_debug_lock); + + hlist_for_each_entry(core, &clk_debug_list, debug_node) + cnt += clock_debug_print_clock(core, s); + + mutex_unlock(&clk_debug_lock); + + if (cnt) + clock_debug_output(s, 0, "Enabled clock count: %d\n", cnt); + else + clock_debug_output(s, 0, "No clocks enabled.\n"); +} + +static int enabled_clocks_show(struct seq_file *s, void *unused) +{ + clock_debug_print_enabled_clocks(s); + + return 0; +} + +static int enabled_clocks_open(struct inode *inode, struct file *file) +{ + return single_open(file, enabled_clocks_show, inode->i_private); +} + +static const struct file_operations clk_enabled_list_fops = { + .open = enabled_clocks_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +void clk_debug_print_hw(struct clk_core *clk, struct seq_file *f) +{ + if (IS_ERR_OR_NULL(clk)) + return; + + clk_debug_print_hw(clk->parent, f); + + clock_debug_output(f, false, "%s\n", clk->name); + + if (!clk->ops->list_registers) + return; + + clk->ops->list_registers(f, clk->hw); +} +EXPORT_SYMBOL(clk_debug_print_hw); + +static int print_hw_show(struct seq_file *m, void *unused) +{ + struct clk_core *c = m->private; + + clk_debug_print_hw(c, m); + + return 0; +} + +static int print_hw_open(struct inode *inode, struct file *file) +{ + return single_open(file, print_hw_show, inode->i_private); +} + +static const struct file_operations clock_print_hw_fops = { + .open = print_hw_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int list_rates_show(struct seq_file *s, void *unused) +{ + struct clk_core *core = s->private; + int level = 0, i = 0; + unsigned long rate, rate_max = 0; + + /* Find max frequency supported within voltage constraints. */ + if (!core->vdd_class) { + rate_max = ULONG_MAX; + } else { + for (level = 0; level < core->num_rate_max; level++) + if (core->rate_max[level]) + rate_max = core->rate_max[level]; + } + + /* + * List supported frequencies <= rate_max. Higher frequencies may + * appear in the frequency table, but are not valid and should not + * be listed. + */ + while (!IS_ERR_VALUE(rate = + core->ops->list_rate(core->hw, i++, rate_max))) { + if (rate <= 0) + break; + if (rate <= rate_max) + seq_printf(s, "%lu\n", rate); + } + + return 0; +} + +static int list_rates_open(struct inode *inode, struct file *file) +{ + return single_open(file, list_rates_show, inode->i_private); +} + +static const struct file_operations list_rates_fops = { + .open = list_rates_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static void clock_print_rate_max_by_level(struct seq_file *s, int level) +{ + struct clk_core *core = s->private; + struct clk_vdd_class *vdd_class = core->vdd_class; + int off, i, vdd_level, nregs = vdd_class->num_regulators; + + vdd_level = clk_find_vdd_level(core, core->rate); + + seq_printf(s, "%2s%10lu", vdd_level == level ? "[" : "", + core->rate_max[level]); + + for (i = 0; i < nregs; i++) { + off = nregs*level + i; + if (vdd_class->vdd_uv) + seq_printf(s, "%10u", vdd_class->vdd_uv[off]); + } + + if (vdd_level == level) + seq_puts(s, "]"); + + seq_puts(s, "\n"); +} + +static int rate_max_show(struct seq_file *s, void *unused) +{ + struct clk_core *core = s->private; + struct clk_vdd_class *vdd_class = core->vdd_class; + int level = 0, i, nregs = vdd_class->num_regulators; + char reg_name[10]; + + int vdd_level = clk_find_vdd_level(core, core->rate); + + if (vdd_level < 0) { + seq_printf(s, "could not find_vdd_level for %s, %ld\n", + core->name, core->rate); + return 0; + } + + seq_printf(s, "%12s", ""); + for (i = 0; i < nregs; i++) { + snprintf(reg_name, ARRAY_SIZE(reg_name), "reg %d", i); + seq_printf(s, "%10s", reg_name); + } + + seq_printf(s, "\n%12s", "freq"); + for (i = 0; i < nregs; i++) + seq_printf(s, "%10s", "uV"); + + seq_puts(s, "\n"); + + for (level = 0; level < core->num_rate_max; level++) + clock_print_rate_max_by_level(s, level); + + return 0; +} + +static int rate_max_open(struct inode *inode, struct file *file) +{ + return single_open(file, rate_max_show, inode->i_private); +} + +static const struct file_operations rate_max_fops = { + .open = rate_max_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + static int clk_debug_create_one(struct clk_core *core, struct dentry *pdentry) { struct dentry *d; @@ -2118,11 +2863,21 @@ core->dentry = d; - d = debugfs_create_u32("clk_rate", S_IRUGO, core->dentry, - (u32 *)&core->rate); + d = debugfs_create_file("clk_rate", S_IRUGO, core->dentry, core, + &clock_rate_fops); if (!d) goto err_out; + if (core->ops->list_rate) { + if (!debugfs_create_file("clk_list_rates", + S_IRUGO, core->dentry, core, &list_rates_fops)) + goto err_out; + } + + if (core->vdd_class && !debugfs_create_file("clk_rate_max", + S_IRUGO, core->dentry, core, &rate_max_fops)) + goto err_out; + d = debugfs_create_u32("clk_accuracy", S_IRUGO, core->dentry, (u32 *)&core->accuracy); if (!d) @@ -2143,8 +2898,8 @@ if (!d) goto err_out; - d = debugfs_create_u32("clk_enable_count", S_IRUGO, core->dentry, - (u32 *)&core->enable_count); + d = debugfs_create_file("clk_enable_count", S_IRUGO, core->dentry, + core, &clock_enable_fops); if (!d) goto err_out; @@ -2153,6 +2908,16 @@ if (!d) goto err_out; + d = debugfs_create_file("clk_parent", S_IRUGO, core->dentry, core, + &clock_parent_fops); + if (!d) + goto err_out; + + d = debugfs_create_file("clk_print_regs", S_IRUGO, core->dentry, + core, &clock_print_hw_fops); + if (!d) + goto err_out; + if (core->ops->debug_init) { ret = core->ops->debug_init(core->hw, core->dentry); if (ret) @@ -2224,6 +2989,19 @@ } EXPORT_SYMBOL_GPL(clk_debugfs_add_file); +/* + * Print the names of all enabled clocks and their parents if + * debug_suspend is set from debugfs. + */ +void clock_debug_print_enabled(void) +{ + if (likely(!debug_suspend)) + return; + + clock_debug_print_enabled_debug_suspend(NULL); +} +EXPORT_SYMBOL_GPL(clock_debug_print_enabled); + /** * clk_debug_init - lazily populate the debugfs clk directory * @@ -2263,6 +3041,22 @@ if (!d) return -ENOMEM; + d = debugfs_create_file("clk_enabled_list", S_IRUGO, rootdir, + &clk_debug_list, &clk_enabled_list_fops); + if (!d) + return -ENOMEM; + + + d = debugfs_create_u32("debug_suspend", S_IRUGO | S_IWUSR, + rootdir, &debug_suspend); + if (!d) + return -ENOMEM; + + d = debugfs_create_file("trace_clocks", S_IRUGO, rootdir, &all_lists, + &clk_state_fops); + if (!d) + return -ENOMEM; + mutex_lock(&clk_debug_lock); hlist_for_each_entry(core, &clk_debug_list, debug_node) clk_debug_create_one(core, rootdir); @@ -2466,6 +3260,47 @@ if (core->ops->init) core->ops->init(core->hw); + if (core->flags & CLK_IS_CRITICAL) { + unsigned long flags; + + clk_core_prepare(core); + + flags = clk_enable_lock(); + clk_core_enable(core); + clk_enable_unlock(flags); + } + + /* + * enable clocks with the CLK_ENABLE_HAND_OFF flag set + * + * This flag causes the framework to enable the clock at registration + * time, which is sometimes necessary for clocks that would cause a + * system crash when gated (e.g. cpu, memory, etc). The prepare_count + * is migrated over to the first clk consumer to call clk_prepare(). + * Similarly the clk's enable_count is migrated to the first consumer + * to call clk_enable(). + */ + if (core->flags & CLK_ENABLE_HAND_OFF) { + unsigned long flags; + + /* + * Few clocks might have hardware gating which would be required + * to be ON before prepare/enabling the clocks. So check if the + * clock has been turned ON earlier and we should + * prepare/enable those clocks. + */ + if (clk_core_is_enabled(core)) { + core->need_handoff_prepare = true; + core->need_handoff_enable = true; + ret = clk_core_prepare(core); + if (ret) + goto out; + flags = clk_enable_lock(); + clk_core_enable(core); + clk_enable_unlock(flags); + } + } + kref_init(&core->ref); out: clk_prepare_unlock(); @@ -2545,8 +3380,19 @@ core->num_parents = hw->init->num_parents; core->min_rate = 0; core->max_rate = ULONG_MAX; + core->vdd_class = hw->init->vdd_class; + core->rate_max = hw->init->rate_max; + core->num_rate_max = hw->init->num_rate_max; hw->core = core; + if (core->vdd_class) { + ret = clk_vdd_class_init(core->vdd_class); + if (ret) { + pr_err("Failed to initialize vdd class\n"); + goto fail_parent_names; + } + } + /* allocate local copy in case parent_names is __initdata */ core->parent_names = kcalloc(core->num_parents, sizeof(char *), GFP_KERNEL); @@ -2901,6 +3747,8 @@ } EXPORT_SYMBOL_GPL(clk_notifier_unregister); +#endif /* CONFIG_COMMON_CLK */ + #ifdef CONFIG_OF /** * struct of_clk_provider - Clock provider registration structure @@ -2931,6 +3779,8 @@ } EXPORT_SYMBOL_GPL(of_clk_src_simple_get); +#if defined(CONFIG_COMMON_CLK) + struct clk *of_clk_src_onecell_get(struct of_phandle_args *clkspec, void *data) { struct clk_onecell_data *clk_data = data; @@ -2945,6 +3795,11 @@ } EXPORT_SYMBOL_GPL(of_clk_src_onecell_get); +#endif /* CONFIG_COMMON_CLK */ + +/* forward declaration */ +void of_clk_del_provider(struct device_node *np); + /** * of_clk_add_provider() - Register a clock provider for a node * @np: Device node pointer associated with clock provider @@ -3100,8 +3955,10 @@ else clk_name = NULL; } else { +#if defined(CONFIG_COMMON_CLK) clk_name = __clk_get_name(clk); clk_put(clk); +#endif } } @@ -3132,6 +3989,8 @@ } EXPORT_SYMBOL_GPL(of_clk_parent_fill); +#if defined(CONFIG_COMMON_CLK) + struct clock_provider { of_clk_init_cb_t clk_init_cb; struct device_node *np; @@ -3174,6 +4033,41 @@ } /** + * of_clk_detect_critical() - set CLK_IS_CRITICAL flag from Device Tree + * @np: Device node pointer associated with clock provider + * @index: clock index + * @flags: pointer to clk_core->flags + * + * Detects if the clock-critical property exists and, if so, sets the + * corresponding CLK_IS_CRITICAL flag. + * + * Do not use this function. It exists only for legacy Device Tree + * bindings, such as the one-clock-per-node style that are outdated. + * Those bindings typically put all clock data into .dts and the Linux + * driver has no clock data, thus making it impossible to set this flag + * correctly from the driver. Only those drivers may call + * of_clk_detect_critical from their setup functions. + * + * Return: error code or zero on success + */ +int of_clk_detect_critical(struct device_node *np, + int index, unsigned long *flags) +{ + struct property *prop; + const __be32 *cur; + uint32_t idx; + + if (!np || !flags) + return -EINVAL; + + of_property_for_each_u32(np, "clock-critical", prop, cur, idx) + if (index == idx) + *flags |= CLK_IS_CRITICAL; + + return 0; +} + +/** * of_clk_init() - Scan and init clock providers from the DT * @matches: array of compatible values and init functions for providers. * @@ -3240,4 +4134,7 @@ force = true; } } + +#endif /* CONFIG_COMMON_CLK */ + #endif diff -ruw linux-4.4.115/drivers/clk/clkdev.c linux-4.4.115-fbx/drivers/clk/clkdev.c --- linux-4.4.115/drivers/clk/clkdev.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clk/clkdev.c 2019-01-22 16:16:23.007241915 +0100 @@ -27,7 +27,7 @@ static LIST_HEAD(clocks); static DEFINE_MUTEX(clocks_mutex); -#if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK) +#if defined(CONFIG_OF) static struct clk *__of_clk_get(struct device_node *np, int index, const char *dev_id, const char *con_id) { @@ -73,14 +73,10 @@ if (name) index = of_property_match_string(np, "clock-names", name); clk = __of_clk_get(np, index, dev_id, name); - if (!IS_ERR(clk)) { + if (!IS_ERR(clk)) break; - } else if (name && index >= 0) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - pr_err("ERROR: could not get clock %s:%s(%i)\n", - np->full_name, name ? name : "", index); + else if (name && index >= 0) return clk; - } /* * No matching clock found on this node. If the parent node @@ -190,7 +186,7 @@ out: mutex_unlock(&clocks_mutex); - return cl ? clk : ERR_PTR(-ENOENT); + return cl ? cl->clk : ERR_PTR(-ENOENT); } EXPORT_SYMBOL(clk_get_sys); diff -ruw linux-4.4.115/drivers/clk/clk.h linux-4.4.115-fbx/drivers/clk/clk.h --- linux-4.4.115/drivers/clk/clk.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clk/clk.h 2019-01-22 16:16:23.007241915 +0100 @@ -11,7 +11,7 @@ struct clk_hw; -#if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK) +#if defined(CONFIG_OF) struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec, const char *dev_id, const char *con_id); #endif @@ -20,6 +20,11 @@ struct clk *__clk_create_clk(struct clk_hw *hw, const char *dev_id, const char *con_id); void __clk_free_clk(struct clk *clk); + +/* Debugfs API to print the enabled clocks */ +void clock_debug_print_enabled(void); +void clk_debug_print_hw(struct clk_core *clk, struct seq_file *f); + #else /* All these casts to avoid ifdefs in clkdev... */ static inline struct clk * diff -ruw linux-4.4.115/drivers/clk/Kconfig linux-4.4.115-fbx/drivers/clk/Kconfig --- linux-4.4.115/drivers/clk/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clk/Kconfig 2019-01-22 16:16:22.995241806 +0100 @@ -198,3 +198,4 @@ source "drivers/clk/samsung/Kconfig" source "drivers/clk/tegra/Kconfig" +source "drivers/clk/msm/Kconfig" diff -ruw linux-4.4.115/drivers/clk/Makefile linux-4.4.115-fbx/drivers/clk/Makefile --- linux-4.4.115/drivers/clk/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clk/Makefile 2019-01-22 16:16:22.995241806 +0100 @@ -1,7 +1,7 @@ # common clock types obj-$(CONFIG_HAVE_CLK) += clk-devres.o obj-$(CONFIG_CLKDEV_LOOKUP) += clkdev.o -obj-$(CONFIG_COMMON_CLK) += clk.o +obj-$(CONFIG_OF) += clk.o obj-$(CONFIG_COMMON_CLK) += clk-divider.o obj-$(CONFIG_COMMON_CLK) += clk-fixed-factor.o obj-$(CONFIG_COMMON_CLK) += clk-fixed-rate.o @@ -84,3 +84,4 @@ obj-$(CONFIG_ARCH_ZX) += zte/ obj-$(CONFIG_ARCH_ZYNQ) += zynq/ obj-$(CONFIG_H8300) += h8300/ +obj-$(CONFIG_ARCH_QCOM) += msm/ diff -ruw linux-4.4.115/drivers/clk/qcom/Kconfig linux-4.4.115-fbx/drivers/clk/qcom/Kconfig --- linux-4.4.115/drivers/clk/qcom/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clk/qcom/Kconfig 2019-01-22 16:16:23.027242096 +0100 @@ -2,6 +2,9 @@ bool select PM_GENERIC_DOMAINS if PM +config QCOM_RPMCC + bool + config COMMON_CLK_QCOM tristate "Support for Qualcomm's clock controllers" depends on OF @@ -9,6 +12,32 @@ select REGMAP_MMIO select RESET_CONTROLLER +config QCOM_CLK_RPM + tristate "RPM based Clock Controller" + depends on COMMON_CLK_QCOM && MFD_QCOM_RPM + select QCOM_RPMCC + help + The RPM (Resource Power Manager) is a dedicated hardware engine for + managing the shared SoC resources in order to keep the lowest power + profile. It communicates with other hardware subsystems via shared + memory and accepts clock requests, aggregates the requests and turns + the clocks on/off or scales them on demand. + Say Y if you want to support the clocks exposed by the RPM on + platforms such as apq8064, msm8660, msm8960 etc. + +config QCOM_CLK_SMD_RPM + tristate "RPM over SMD based Clock Controller" + depends on COMMON_CLK_QCOM + select QCOM_RPMCC + help + The RPM (Resource Power Manager) is a dedicated hardware engine for + managing the shared SoC resources in order to keep the lowest power + profile. It communicates with other hardware subsystems via shared + memory and accepts clock requests, aggregates the requests and turns + the clocks on/off or scales them on demand. + Say Y if you want to support the clocks exposed by the RPM on + platforms such as apq8016, apq8084, msm8974 etc. + config APQ_GCC_8084 tristate "APQ8084 Global Clock Controller" select QCOM_GDSC @@ -106,3 +135,109 @@ Support for the multimedia clock controller on msm8974 devices. Say Y if you want to support multimedia devices such as display, graphics, video encode/decode, camera, etc. + +config MSM_GCC_8996 + tristate "MSM8996 Global Clock Controller" + depends on COMMON_CLK_QCOM + help + Support for the global clock controller on msm8996 devices. + Say Y if you want to use peripheral devices such as UART, SPI, + i2c, USB, UFS, SD/eMMC, PCIe, etc. + +config MSM_MMCC_8996 + tristate "MSM8996 Multimedia Clock Controller" + select MSM_GCC_8996 + depends on COMMON_CLK_QCOM + help + Support for the multimedia clock controller on msm8996 devices. + Say Y if you want to support multimedia devices such as display, + graphics, video encode/decode, camera, etc. + +config MSM_GCC_660 + tristate "SDM660 Global Clock Controller" + select QCOM_GDSC + depends on COMMON_CLK_QCOM + ---help--- + Support for the global clock controller on Qualcomm Technologies, Inc + SDM660 devices. + Say Y if you want to use peripheral devices such as UART, SPI, I2C, + USB, UFS, SD/eMMC, PCIe, etc. + +config MSM_GPUCC_660 + tristate "SDM660 Graphics Clock Controller" + select MSM_GCC_660 + depends on COMMON_CLK_QCOM + help + Support for the graphics clock controller on Qualcomm Technologies, Inc + SDM660 devices. + Say Y if you want to support graphics controller devices which will + be required to enable those device. + +config MSM_MMCC_660 + tristate "SDM660 Multimedia Clock Controller" + select MSM_GCC_660 + depends on COMMON_CLK_QCOM + help + Support for the multimedia clock controller on Qualcomm Technologies, Inc + SDM660 devices. + Say Y if you want to support multimedia devices such as display, + video encode/decode, camera, etc. + +config QCOM_HFPLL + tristate "High-Frequency PLL (HFPLL) Clock Controller" + depends on COMMON_CLK_QCOM + help + Support for the high-frequency PLLs present on MSM devices. + Say Y if you want to support CPU frequency scaling on devices + such as MSM8974, APQ8084, etc. + +config KPSS_XCC + tristate "KPSS Clock Controller" + depends on COMMON_CLK_QCOM + help + Support for the Krait ACC and GCC clock controllers. Say Y + if you want to support CPU frequency scaling on devices such + as MSM8960, APQ8064, etc. + +config KRAITCC + tristate "Krait Clock Controller" + depends on COMMON_CLK_QCOM && ARM + select KRAIT_CLOCKS + help + Support for the Krait CPU clocks on MSM devices. + Say Y if you want to support CPU frequency scaling. + +config KRAIT_CLOCKS + bool + select KRAIT_L2_ACCESSORS + +config QCOM_A53 + tristate "A53 Clock Controller" + depends on COMMON_CLK_QCOM + help + Support for the A53 clock controller on MSM devices. + Say Y if you want to support CPU frequency scaling on devices + such as MSM8916. + +config CLOCK_CPU_OSM + tristate "OSM CPU Clock Controller" + depends on COMMON_CLK_QCOM + help + Support for the osm clock controller. + Operating State Manager (OSM) is a hardware engine used by some + Qualcomm Technologies, Inc. (QTI) SoCs to manage frequency and + voltage scaling in hardware. OSM is capable of controlling + frequency and voltage requests for multiple clusters via the + existence of multiple OSM domains. + Say Y if you want to support osm clocks. + +config CLOCK_QPNP_DIV + tristate "QPNP PMIC clkdiv driver" + depends on COMMON_CLK_QCOM && SPMI + help + This driver supports the clkdiv functionality on the Qualcomm + Technologies, Inc. QPNP PMIC. It configures the frequency of + clkdiv outputs on the PMIC. These clocks are typically wired + through alternate functions on gpio pins. + +source "drivers/clk/qcom/mdss/Kconfig" diff -ruw linux-4.4.115/drivers/clk/qcom/Makefile linux-4.4.115-fbx/drivers/clk/qcom/Makefile --- linux-4.4.115/drivers/clk/qcom/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clk/qcom/Makefile 2019-01-22 16:16:23.027242096 +0100 @@ -2,14 +2,19 @@ clk-qcom-y += common.o clk-qcom-y += clk-regmap.o +clk-qcom-y += clk-alpha-pll.o clk-qcom-y += clk-pll.o clk-qcom-y += clk-rcg.o clk-qcom-y += clk-rcg2.o clk-qcom-y += clk-branch.o clk-qcom-y += clk-regmap-divider.o clk-qcom-y += clk-regmap-mux.o -clk-qcom-y += reset.o -clk-qcom-$(CONFIG_QCOM_GDSC) += gdsc.o +clk-qcom-y += clk-regmap-mux-div.o +clk-qcom-$(CONFIG_KRAIT_CLOCKS) += clk-krait.o +clk-qcom-y += clk-hfpll.o +clk-qcom-y += reset.o clk-voter.o +clk-qcom-y += clk-dummy.o clk-debug.o +clk-qcom-$(CONFIG_QCOM_GDSC) += gdsc.o gdsc-regulator.o obj-$(CONFIG_APQ_GCC_8084) += gcc-apq8084.o obj-$(CONFIG_APQ_MMCC_8084) += mmcc-apq8084.o @@ -20,5 +25,20 @@ obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o +obj-$(CONFIG_MSM_GCC_8996) += gcc-msm8996.o +obj-$(CONFIG_MSM_GCC_660) += gcc-sdm660.o obj-$(CONFIG_MSM_MMCC_8960) += mmcc-msm8960.o obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o +obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o +obj-$(CONFIG_MSM_GPUCC_660) += gpucc-sdm660.o +obj-$(CONFIG_MSM_MMCC_660) += mmcc-sdm660.o +obj-$(CONFIG_KPSS_XCC) += kpss-xcc.o +obj-$(CONFIG_QCOM_HFPLL) += hfpll.o +obj-$(CONFIG_KRAITCC) += krait-cc.o +obj-$(CONFIG_QCOM_A53) += clk-a53.o +obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o +obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o +obj-$(CONFIG_CLOCK_CPU_OSM) += clk-cpu-osm.o +obj-$(CONFIG_CLOCK_QPNP_DIV) += clk-qpnp-div.o + +obj-y += mdss/ diff -ruw linux-4.4.115/drivers/clocksource/arm_arch_timer.c linux-4.4.115-fbx/drivers/clocksource/arm_arch_timer.c --- linux-4.4.115/drivers/clocksource/arm_arch_timer.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clocksource/arm_arch_timer.c 2019-10-29 09:26:23.489201671 +0100 @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -83,20 +84,20 @@ struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: - writel_relaxed(val, timer->base + CNTP_CTL); + writel_relaxed_no_log(val, timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: - writel_relaxed(val, timer->base + CNTP_TVAL); + writel_relaxed_no_log(val, timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: - writel_relaxed(val, timer->base + CNTV_CTL); + writel_relaxed_no_log(val, timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: - writel_relaxed(val, timer->base + CNTV_TVAL); + writel_relaxed_no_log(val, timer->base + CNTV_TVAL); break; } } else { @@ -114,20 +115,20 @@ struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: - val = readl_relaxed(timer->base + CNTP_CTL); + val = readl_relaxed_no_log(timer->base + CNTP_CTL); break; case ARCH_TIMER_REG_TVAL: - val = readl_relaxed(timer->base + CNTP_TVAL); + val = readl_relaxed_no_log(timer->base + CNTP_TVAL); break; } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); switch (reg) { case ARCH_TIMER_REG_CTRL: - val = readl_relaxed(timer->base + CNTV_CTL); + val = readl_relaxed_no_log(timer->base + CNTV_CTL); break; case ARCH_TIMER_REG_TVAL: - val = readl_relaxed(timer->base + CNTV_TVAL); + val = readl_relaxed_no_log(timer->base + CNTV_TVAL); break; } } else { @@ -328,12 +329,16 @@ /* Disable user access to the timers and the physical counter */ /* Also disable virtual event stream */ cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN - | ARCH_TIMER_USR_VT_ACCESS_EN | ARCH_TIMER_VIRT_EVT_EN | ARCH_TIMER_USR_PCT_ACCESS_EN); /* Enable user access to the virtual counter */ + cntkctl |= ARCH_TIMER_USR_VT_ACCESS_EN; + + if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_VCT_ACCESS)) cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; + else + cntkctl &= ~ARCH_TIMER_USR_VCT_ACCESS_EN; arch_timer_set_cntkctl(cntkctl); } @@ -371,7 +376,8 @@ if (!acpi_disabled || of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) { if (cntbase) - arch_timer_rate = readl_relaxed(cntbase + CNTFRQ); + arch_timer_rate = readl_relaxed_no_log(cntbase + + CNTFRQ); else arch_timer_rate = arch_timer_get_cntfrq(); } @@ -408,9 +414,9 @@ u32 vct_lo, vct_hi, tmp_hi; do { - vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); - vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO); - tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI); + vct_hi = readl_relaxed_no_log(arch_counter_base + CNTVCT_HI); + vct_lo = readl_relaxed_no_log(arch_counter_base + CNTVCT_LO); + tmp_hi = readl_relaxed_no_log(arch_counter_base + CNTVCT_HI); } while (vct_hi != tmp_hi); return ((u64) vct_hi << 32) | vct_lo; @@ -686,6 +692,7 @@ arch_timer_banner(arch_timers_present); arch_counter_register(arch_timers_present); arch_timer_arch_init(); + clocksource_select_force(); } static void __init arch_timer_init(void) @@ -756,7 +763,7 @@ return; } - cnttidr = readl_relaxed(cntctlbase + CNTTIDR); + cnttidr = readl_relaxed_no_log(cntctlbase + CNTTIDR); iounmap(cntctlbase); /* @@ -804,6 +811,7 @@ arch_timer_detect_rate(base, np); arch_timer_mem_register(base, irq); arch_timer_common_init(); + get_pct_hook_init(); } CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem", arch_timer_mem_init); diff -ruw linux-4.4.115/drivers/clocksource/Kconfig linux-4.4.115-fbx/drivers/clocksource/Kconfig --- linux-4.4.115/drivers/clocksource/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/clocksource/Kconfig 2019-01-22 16:16:23.067242458 +0100 @@ -158,6 +158,23 @@ This must be disabled for hardware validation purposes to detect any hardware anomalies of missing events. +config MSM_TIMER_LEAP + bool "ARCH TIMER counter rollover" + default n + depends on ARM_ARCH_TIMER && ARM64 + help + This option enables a check for least significant 32 bits of + counter rollover. On every counter read if least significant + 32 bits are set, reread counter. + +config ARM_ARCH_TIMER_VCT_ACCESS + bool "Support for ARM architected timer virtual counter access in userspace" + default !ARM64 + depends on ARM_ARCH_TIMER + help + This option enables support for reading the ARM architected timer's + virtual counter in userspace. + config ARM_GLOBAL_TIMER bool select CLKSRC_OF if OF diff -ruw linux-4.4.115/drivers/cpufreq/cpufreq.c linux-4.4.115-fbx/drivers/cpufreq/cpufreq.c --- linux-4.4.115/drivers/cpufreq/cpufreq.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/cpufreq/cpufreq.c 2019-10-29 09:26:23.493201710 +0100 @@ -29,6 +29,9 @@ #include #include #include +#ifdef CONFIG_SMP +#include +#endif #include static LIST_HEAD(cpufreq_policy_list); @@ -127,6 +130,7 @@ */ static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); static struct srcu_notifier_head cpufreq_transition_notifier_list; +struct atomic_notifier_head cpufreq_govinfo_notifier_list; static bool init_cpufreq_transition_notifier_list_called; static int __init init_cpufreq_transition_notifier_list(void) @@ -137,6 +141,15 @@ } pure_initcall(init_cpufreq_transition_notifier_list); +static bool init_cpufreq_govinfo_notifier_list_called; +static int __init init_cpufreq_govinfo_notifier_list(void) +{ + ATOMIC_INIT_NOTIFIER_HEAD(&cpufreq_govinfo_notifier_list); + init_cpufreq_govinfo_notifier_list_called = true; + return 0; +} +pure_initcall(init_cpufreq_govinfo_notifier_list); + static int off __read_mostly; static int cpufreq_disabled(void) { @@ -154,6 +167,12 @@ } EXPORT_SYMBOL_GPL(have_governor_per_policy); +bool cpufreq_driver_is_slow(void) +{ + return !(cpufreq_driver->flags & CPUFREQ_DRIVER_FAST); +} +EXPORT_SYMBOL_GPL(cpufreq_driver_is_slow); + struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) { if (have_governor_per_policy()) @@ -347,6 +366,50 @@ #endif } +/********************************************************************* + * FREQUENCY INVARIANT CPU CAPACITY * + *********************************************************************/ + +static DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; +static DEFINE_PER_CPU(unsigned long, max_freq_scale) = SCHED_CAPACITY_SCALE; + +static void +scale_freq_capacity(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) +{ + unsigned long cur = freqs ? freqs->new : policy->cur; + unsigned long scale = (cur << SCHED_CAPACITY_SHIFT) / policy->max; + struct cpufreq_cpuinfo *cpuinfo = &policy->cpuinfo; + int cpu; + + pr_debug("cpus %*pbl cur/cur max freq %lu/%u kHz freq scale %lu\n", + cpumask_pr_args(policy->cpus), cur, policy->max, scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(freq_scale, cpu) = scale; + + if (freqs) + return; + + scale = (policy->max << SCHED_CAPACITY_SHIFT) / cpuinfo->max_freq; + + pr_debug("cpus %*pbl cur max/max freq %u/%u kHz max freq scale %lu\n", + cpumask_pr_args(policy->cpus), policy->max, cpuinfo->max_freq, + scale); + + for_each_cpu(cpu, policy->cpus) + per_cpu(max_freq_scale, cpu) = scale; +} + +unsigned long cpufreq_scale_freq_capacity(struct sched_domain *sd, int cpu) +{ + return per_cpu(freq_scale, cpu); +} + +unsigned long cpufreq_scale_max_freq_capacity(int cpu) +{ + return per_cpu(max_freq_scale, cpu); +} + static void __cpufreq_notify_transition(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs, unsigned int state) { @@ -423,6 +486,9 @@ void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, struct cpufreq_freqs *freqs) { +#ifdef CONFIG_SMP + int cpu; +#endif /* * Catch double invocations of _begin() which lead to self-deadlock. @@ -450,6 +516,12 @@ spin_unlock(&policy->transition_lock); + scale_freq_capacity(policy, freqs); +#ifdef CONFIG_SMP + for_each_cpu(cpu, policy->cpus) + trace_cpu_capacity(capacity_curr_of(cpu), cpu); +#endif + cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); } EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); @@ -469,6 +541,38 @@ } EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); +/** + * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported + * one. + * @target_freq: target frequency to resolve. + * + * The target to driver frequency mapping is cached in the policy. + * + * Return: Lowest driver-supported frequency greater than or equal to the + * given target_freq, subject to policy (min/max) and driver limitations. + */ +unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + target_freq = clamp_val(target_freq, policy->min, policy->max); + policy->cached_target_freq = target_freq; + + if (cpufreq_driver->target_index) { + int idx, rv; + + rv = cpufreq_frequency_table_target(policy, policy->freq_table, + target_freq, + CPUFREQ_RELATION_L, + &idx); + if (rv) + return target_freq; + policy->cached_resolved_idx = idx; + return policy->freq_table[idx].frequency; + } + + return target_freq; +} +EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); /********************************************************************* * SYSFS INTERFACE * @@ -1001,7 +1105,8 @@ if (has_target()) { ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); if (ret) { - pr_err("%s: Failed to stop governor\n", __func__); + pr_err("%s: Failed to stop governor for CPU%u, policy CPU%u\n", + __func__, cpu, policy->cpu); return ret; } } @@ -1016,7 +1121,8 @@ ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); if (ret) { - pr_err("%s: Failed to start governor\n", __func__); + pr_err("%s: Failed to start governor for CPU%u, policy CPU%u\n", + __func__, cpu, policy->cpu); return ret; } } @@ -1329,7 +1435,8 @@ if (has_target()) { int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); if (ret) - pr_err("%s: Failed to stop governor\n", __func__); + pr_err("%s: Failed to stop governor for CPU%u\n", + __func__, cpu); } down_write(&policy->rwsem); @@ -1379,7 +1486,8 @@ if (has_target()) { int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); if (ret) - pr_err("%s: Failed to exit governor\n", __func__); + pr_err("%s: Failed to start governor for CPU%u, policy CPU%u\n", + __func__, cpu, policy->cpu); } /* @@ -1708,7 +1816,8 @@ if (cpufreq_disabled()) return -EINVAL; - WARN_ON(!init_cpufreq_transition_notifier_list_called); + WARN_ON(!init_cpufreq_transition_notifier_list_called || + !init_cpufreq_govinfo_notifier_list_called); switch (list) { case CPUFREQ_TRANSITION_NOTIFIER: @@ -1719,6 +1828,10 @@ ret = blocking_notifier_chain_register( &cpufreq_policy_notifier_list, nb); break; + case CPUFREQ_GOVINFO_NOTIFIER: + ret = atomic_notifier_chain_register( + &cpufreq_govinfo_notifier_list, nb); + break; default: ret = -EINVAL; } @@ -1753,6 +1866,10 @@ ret = blocking_notifier_chain_unregister( &cpufreq_policy_notifier_list, nb); break; + case CPUFREQ_GOVINFO_NOTIFIER: + ret = atomic_notifier_chain_unregister( + &cpufreq_govinfo_notifier_list, nb); + break; default: ret = -EINVAL; } @@ -1865,15 +1982,6 @@ pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", policy->cpu, target_freq, relation, old_target_freq); - /* - * This might look like a redundant call as we are checking it again - * after finding index. But it is left intentionally for cases where - * exactly same freq is called again and so we can save on few function - * calls. - */ - if (target_freq == policy->cur) - return 0; - /* Save last value to restore later on errors */ policy->restore_freq = policy->cur; @@ -2131,8 +2239,11 @@ blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_NOTIFY, new_policy); + scale_freq_capacity(new_policy, NULL); + policy->min = new_policy->min; policy->max = new_policy->max; + trace_cpu_frequency_limits(policy->max, policy->min, policy->cpu); pr_debug("new min and max freqs are %u - %u kHz\n", policy->min, policy->max); @@ -2259,6 +2370,9 @@ { unsigned int cpu = (unsigned long)hcpu; + if (!cpufreq_driver) + return NOTIFY_OK; + switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: cpufreq_online(cpu); @@ -2425,6 +2539,9 @@ pr_debug("trying to register driver %s\n", driver_data->name); + /* Register for hotplug notifers before blocking hotplug. */ + register_hotcpu_notifier(&cpufreq_cpu_notifier); + /* Protect against concurrent CPU online/offline. */ get_online_cpus(); @@ -2457,8 +2574,7 @@ goto err_if_unreg; } - register_hotcpu_notifier(&cpufreq_cpu_notifier); - pr_debug("driver %s up and running\n", driver_data->name); + pr_info("driver %s up and running\n", driver_data->name); out: put_online_cpus(); @@ -2491,7 +2607,7 @@ if (!cpufreq_driver || (driver != cpufreq_driver)) return -EINVAL; - pr_debug("unregistering driver %s\n", driver->name); + pr_info("unregistering driver %s\n", driver->name); /* Protect against concurrent cpu hotplug */ get_online_cpus(); diff -ruw linux-4.4.115/drivers/cpufreq/Kconfig linux-4.4.115-fbx/drivers/cpufreq/Kconfig --- linux-4.4.115/drivers/cpufreq/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/cpufreq/Kconfig 2019-10-29 09:26:23.493201710 +0100 @@ -102,6 +102,33 @@ Be aware that not all cpufreq drivers support the conservative governor. If unsure have a look at the help section of the driver. Fallback governor will be the performance governor. + +config CPU_FREQ_DEFAULT_GOV_INTERACTIVE + bool "interactive" + select CPU_FREQ_GOV_INTERACTIVE + help + Use the CPUFreq governor 'interactive' as default. This allows + you to get a full dynamic cpu frequency capable system by simply + loading your cpufreq low-level hardware driver, using the + 'interactive' governor for latency-sensitive workloads. + +config CPU_FREQ_DEFAULT_GOV_SCHED + bool "sched" + select CPU_FREQ_GOV_SCHED + help + Use the CPUfreq governor 'sched' as default. This scales + cpu frequency using CPU utilization estimates from the + scheduler. + +config CPU_FREQ_DEFAULT_GOV_SCHEDUTIL + bool "schedutil" + depends on SMP + select CPU_FREQ_GOV_SCHEDUTIL + select CPU_FREQ_GOV_PERFORMANCE + help + Use the 'schedutil' CPUFreq governor by default. If unsure, + have a look at the help section of that governor. The fallback + governor will be 'performance'. endchoice config CPU_FREQ_GOV_PERFORMANCE @@ -159,6 +186,20 @@ If in doubt, say N. +config CPU_FREQ_GOV_INTERACTIVE + bool "'interactive' cpufreq policy governor" + help + 'interactive' - This driver adds a dynamic cpufreq policy governor + designed for latency-sensitive workloads. + + This governor attempts to reduce the latency of clock + increases so that the system is more responsive to + interactive workloads. + + For details, take a look at linux/Documentation/cpu-freq. + + If in doubt, say N. + config CPU_FREQ_GOV_CONSERVATIVE tristate "'conservative' cpufreq governor" depends on CPU_FREQ @@ -183,6 +224,34 @@ If in doubt, say N. +config CPU_BOOST + tristate "Event base short term CPU freq boost" + depends on CPU_FREQ + help + This driver boosts the frequency of one or more CPUs based on + various events that might occur in the system. As of now, the + events it reacts to are: + - Migration of important threads from one CPU to another. + + If in doubt, say N. + +config CPU_FREQ_GOV_SCHEDUTIL + bool "'schedutil' cpufreq policy governor" + depends on CPU_FREQ && SMP + select CPU_FREQ_GOV_ATTR_SET + select IRQ_WORK + help + This governor makes decisions based on the utilization data provided + by the scheduler. It sets the CPU frequency to be proportional to + the utilization/capacity ratio coming from the scheduler. If the + utilization is frequency-invariant, the new frequency is also + proportional to the maximum available frequency. If that is not the + case, it is proportional to the current frequency of the CPU. The + frequency tipping point is at utilization/capacity equal to 80% in + both cases. + + If in doubt, say N. + comment "CPU frequency scaling drivers" config CPUFREQ_DT diff -ruw linux-4.4.115/drivers/cpufreq/Kconfig.arm linux-4.4.115-fbx/drivers/cpufreq/Kconfig.arm --- linux-4.4.115/drivers/cpufreq/Kconfig.arm 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/cpufreq/Kconfig.arm 2019-01-22 16:16:23.075242531 +0100 @@ -255,3 +255,10 @@ support for its operation. If in doubt, say N. + +config CPU_FREQ_MSM + bool "MSM CPUFreq support" + depends on CPU_FREQ + default y + help + This enables the CPUFreq driver for Qualcomm CPUs. diff -ruw linux-4.4.115/drivers/cpufreq/Makefile linux-4.4.115-fbx/drivers/cpufreq/Makefile --- linux-4.4.115/drivers/cpufreq/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/cpufreq/Makefile 2019-10-29 09:26:23.493201710 +0100 @@ -1,5 +1,5 @@ # CPUfreq core -obj-$(CONFIG_CPU_FREQ) += cpufreq.o freq_table.o +obj-$(CONFIG_CPU_FREQ) += cpufreq.o freq_table.o cpufreq_governor_attr_set.o # CPUfreq stats obj-$(CONFIG_CPU_FREQ_STAT) += cpufreq_stats.o @@ -10,7 +10,9 @@ obj-$(CONFIG_CPU_FREQ_GOV_USERSPACE) += cpufreq_userspace.o obj-$(CONFIG_CPU_FREQ_GOV_ONDEMAND) += cpufreq_ondemand.o obj-$(CONFIG_CPU_FREQ_GOV_CONSERVATIVE) += cpufreq_conservative.o +obj-$(CONFIG_CPU_FREQ_GOV_INTERACTIVE) += cpufreq_interactive.o obj-$(CONFIG_CPU_FREQ_GOV_COMMON) += cpufreq_governor.o +obj-$(CONFIG_CPU_BOOST) += cpu-boost.o obj-$(CONFIG_CPUFREQ_DT) += cpufreq-dt.o @@ -77,7 +79,7 @@ obj-$(CONFIG_ARM_TEGRA124_CPUFREQ) += tegra124-cpufreq.o obj-$(CONFIG_ARM_VEXPRESS_SPC_CPUFREQ) += vexpress-spc-cpufreq.o obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o - +obj-$(CONFIG_CPU_FREQ_MSM) += qcom-cpufreq.o ################################################################################## # PowerPC platform drivers diff -ruw linux-4.4.115/drivers/cpuidle/cpuidle.c linux-4.4.115-fbx/drivers/cpuidle/cpuidle.c --- linux-4.4.115/drivers/cpuidle/cpuidle.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/cpuidle/cpuidle.c 2019-01-22 16:16:23.095242712 +0100 @@ -193,7 +193,7 @@ } /* Take note of the planned idle state. */ - sched_idle_set_state(target_state); + sched_idle_set_state(target_state, index); trace_cpu_idle_rcuidle(index, dev->cpu); time_start = ktime_get(); @@ -206,7 +206,7 @@ trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu); /* The cpu is no longer idle or about to enter idle. */ - sched_idle_set_state(NULL); + sched_idle_set_state(NULL, -1); if (broadcast) { if (WARN_ON_ONCE(!irqs_disabled())) @@ -614,16 +614,31 @@ #ifdef CONFIG_SMP +static void smp_callback(void *v) +{ + /* we already woke the CPU up, nothing more to do */ +} + /* * This function gets called when a part of the kernel has a new latency - * requirement. This means we need to get all processors out of their C-state, - * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that - * wakes them all right up. + * requirement. This means we need to get only those processors out of their + * C-state for which qos requirement is changed, and then recalculate a new + * suitable C-state. Just do a cross-cpu IPI; that wakes them all right up. */ static int cpuidle_latency_notify(struct notifier_block *b, unsigned long l, void *v) { - wake_up_all_idle_cpus(); + struct cpumask cpus; + + if (v) + cpumask_andnot(&cpus, v, cpu_isolated_mask); + else + cpumask_andnot(&cpus, cpu_online_mask, cpu_isolated_mask); + + preempt_disable(); + smp_call_function_many(&cpus, smp_callback, NULL, 1); + preempt_enable(); + return NOTIFY_OK; } diff -ruw linux-4.4.115/drivers/cpuidle/governors/menu.c linux-4.4.115-fbx/drivers/cpuidle/governors/menu.c --- linux-4.4.115/drivers/cpuidle/governors/menu.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/cpuidle/governors/menu.c 2019-01-22 16:16:23.095242712 +0100 @@ -178,7 +178,12 @@ /* for higher loadavg, we are more reluctant */ - mult += 2 * get_loadavg(load); + /* + * this doesn't work as intended - it is almost always 0, but can + * sometimes, depending on workload, spike very high into the hundreds + * even when the average cpu load is under 10%. + */ + /* mult += 2 * get_loadavg(); */ /* for IO wait tasks (per cpu!) we add 5x each */ mult += 10 * nr_iowaiters; diff -ruw linux-4.4.115/drivers/cpuidle/Makefile linux-4.4.115-fbx/drivers/cpuidle/Makefile --- linux-4.4.115/drivers/cpuidle/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/cpuidle/Makefile 2019-01-22 16:16:23.091242676 +0100 @@ -27,3 +27,4 @@ # POWERPC drivers obj-$(CONFIG_PSERIES_CPUIDLE) += cpuidle-pseries.o obj-$(CONFIG_POWERNV_CPUIDLE) += cpuidle-powernv.o +obj-$(CONFIG_MSM_PM) += lpm-levels.o lpm-levels-of.o lpm-workarounds.o diff -ruw linux-4.4.115/drivers/crypto/Kconfig linux-4.4.115-fbx/drivers/crypto/Kconfig --- linux-4.4.115/drivers/crypto/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/crypto/Kconfig 2019-01-22 16:16:23.095242712 +0100 @@ -346,6 +346,64 @@ Select this to offload Samsung S5PV210 or S5PC110, Exynos from AES algorithms execution. +config CRYPTO_DEV_QCE50 + bool + +config FIPS_ENABLE + bool "FIPS140-2 compliant build" + default n + help + This flag is used to make current build FIPS140-2 + compliant. This flag will enable the patch of code + which will perform this task. Please select Y here + to enable. + +config CRYPTO_DEV_QCRYPTO + tristate "Qualcomm Crypto accelerator" + select CRYPTO_DES + select CRYPTO_ALGAPI + select CRYPTO_AUTHENC + select CRYPTO_BLKCIPHER + default n + help + This driver supports Qualcomm crypto acceleration. + To compile this driver as a module, choose M here: the + module will be called qcrypto. + +config CRYPTO_DEV_QCOM_MSM_QCE + tristate "Qualcomm Crypto Engine (QCE) module" + select CRYPTO_DEV_QCE50 if ARCH_APQ8084 || ARCH_MSM8916 || ARCH_MSM8994 || ARCH_MSM8996 || ARCH_MSM8992 || ARCH_MSMTITANIUM || ARCH_MSM8909 || ARCH_MSM8998 || ARCH_SDM660 || ARCH_SDM630 + default n + help + This driver supports Qualcomm Crypto Engine in MSM7x30, MSM8660 + MSM8x55, MSM8960, MSM9615, MSM8916, MSM8994, MSM8996, FSM9900, + MSMTITANINUM, APQ8084, MSM8998, SDM660 and SDM630. + + To compile this driver as a module, choose M here: the + For MSM7x30 MSM8660 and MSM8x55 the module is called qce + For MSM8960, APQ8064 and MSM9615 the module is called qce40 + For MSM8974, MSM8916, MSM8994, MSM8996, MSM8992, MSMTITANIUM, + APQ8084, MSM8998, SDM660 and SDM630 the module is called qce50. + +config CRYPTO_DEV_QCEDEV + tristate "QCEDEV Interface to CE module" + default n + help + This driver supports Qualcomm QCEDEV Crypto in MSM7x30, MSM8660, + MSM8960, MSM9615, APQ8064, MSM8974, MSM8916, MSM8994, MSM8996, + APQ8084, MSM8998, SDM660, SDM630. This exposes the + interface to the QCE hardware accelerator via IOCTLs. + + To compile this driver as a module, choose M here: the + module will be called qcedev. + +config CRYPTO_DEV_OTA_CRYPTO + tristate "OTA Crypto module" + help + This driver supports Qualcomm OTA Crypto in the FSM9xxx. + To compile this driver as a module, choose M here: the + module will be called ota_crypto. + config CRYPTO_DEV_NX bool "Support for IBM PowerPC Nest (NX) cryptographic acceleration" depends on PPC64 @@ -498,4 +556,9 @@ To compile this driver as a module, choose M here: the module will be called sun4i-ss. +if ARCH_QCOM +source drivers/crypto/msm/Kconfig +endif # ARCH_QCOM + + endif # CRYPTO_HW diff -ruw linux-4.4.115/drivers/crypto/Makefile linux-4.4.115-fbx/drivers/crypto/Makefile --- linux-4.4.115/drivers/crypto/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/crypto/Makefile 2019-01-22 16:16:23.095242712 +0100 @@ -29,3 +29,4 @@ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/ +obj-$(CONFIG_ARCH_QCOM) += msm/ diff -ruw linux-4.4.115/drivers/devfreq/devfreq.c linux-4.4.115-fbx/drivers/devfreq/devfreq.c --- linux-4.4.115/drivers/devfreq/devfreq.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/devfreq.c 2019-10-29 09:26:23.513201906 +0100 @@ -69,11 +69,34 @@ } /** + * devfreq_set_freq_limits() - Set min and max frequency from freq_table + * @devfreq: the devfreq instance + */ +static void devfreq_set_freq_limits(struct devfreq *devfreq) +{ + int idx; + unsigned long min = ~0, max = 0; + + if (!devfreq->profile->freq_table) + return; + + for (idx = 0; idx < devfreq->profile->max_state; idx++) { + if (min > devfreq->profile->freq_table[idx]) + min = devfreq->profile->freq_table[idx]; + if (max < devfreq->profile->freq_table[idx]) + max = devfreq->profile->freq_table[idx]; + } + + devfreq->min_freq = min; + devfreq->max_freq = max; +} + +/** * devfreq_get_freq_level() - Lookup freq_table for the frequency * @devfreq: the devfreq instance * @freq: the target frequency */ -static int devfreq_get_freq_level(struct devfreq *devfreq, unsigned long freq) +int devfreq_get_freq_level(struct devfreq *devfreq, unsigned long freq) { int lev; @@ -83,6 +106,7 @@ return -EINVAL; } +EXPORT_SYMBOL(devfreq_get_freq_level); /** * devfreq_update_status() - Update statistics of devfreq behavior @@ -172,7 +196,7 @@ return -EINVAL; /* Reevaluate the proper frequency */ - err = devfreq->governor->get_target_freq(devfreq, &freq); + err = devfreq->governor->get_target_freq(devfreq, &freq, &flags); if (err) return err; @@ -486,6 +510,7 @@ devfreq->profile->max_state, GFP_KERNEL); devfreq->last_stat_updated = jiffies; + devfreq_set_freq_limits(devfreq); dev_set_name(&devfreq->dev, "%s", dev_name(dev)); err = device_register(&devfreq->dev); @@ -536,7 +561,6 @@ return -EINVAL; device_unregister(&devfreq->dev); - put_device(&devfreq->dev); return 0; } @@ -782,7 +806,7 @@ struct devfreq *df = to_devfreq(dev); int ret; char str_governor[DEVFREQ_NAME_LEN + 1]; - struct devfreq_governor *governor; + const struct devfreq_governor *governor, *prev_gov; ret = sscanf(buf, "%" __stringify(DEVFREQ_NAME_LEN) "s", str_governor); if (ret != 1) @@ -807,12 +831,21 @@ goto out; } } + prev_gov = df->governor; df->governor = governor; strncpy(df->governor_name, governor->name, DEVFREQ_NAME_LEN); ret = df->governor->event_handler(df, DEVFREQ_GOV_START, NULL); - if (ret) + if (ret) { dev_warn(dev, "%s: Governor %s not started(%d)\n", __func__, df->governor->name, ret); + if (prev_gov) { + df->governor = prev_gov; + strncpy(df->governor_name, prev_gov->name, + DEVFREQ_NAME_LEN); + df->governor->event_handler(df, DEVFREQ_GOV_START, + NULL); + } + } out: mutex_unlock(&devfreq_list_lock); @@ -969,19 +1002,26 @@ struct devfreq *df = to_devfreq(d); struct device *dev = df->dev.parent; struct dev_pm_opp *opp; + unsigned int i = 0, max_state = df->profile->max_state; + bool use_opp; ssize_t count = 0; unsigned long freq = 0; rcu_read_lock(); - do { + use_opp = dev_pm_opp_get_opp_count(dev) > 0; + while (use_opp || (!use_opp && i < max_state)) { + if (use_opp) { opp = dev_pm_opp_find_freq_ceil(dev, &freq); if (IS_ERR(opp)) break; + } else { + freq = df->profile->freq_table[i++]; + } count += scnprintf(&buf[count], (PAGE_SIZE - count - 2), "%lu ", freq); freq++; - } while (1); + } rcu_read_unlock(); /* Truncate the trailing space */ diff -ruw linux-4.4.115/drivers/devfreq/governor.h linux-4.4.115-fbx/drivers/devfreq/governor.h --- linux-4.4.115/drivers/devfreq/governor.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/governor.h 2019-01-22 16:16:23.135243074 +0100 @@ -4,6 +4,8 @@ * Copyright (C) 2011 Samsung Electronics * MyungJoo Ham * + * Copyright (c) 2018 The Linux Foundation. All rights reserved. + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. @@ -25,9 +27,6 @@ #define DEVFREQ_GOV_SUSPEND 0x4 #define DEVFREQ_GOV_RESUME 0x5 -/* Caution: devfreq->lock must be locked before calling update_devfreq */ -extern int update_devfreq(struct devfreq *devfreq); - extern void devfreq_monitor_start(struct devfreq *devfreq); extern void devfreq_monitor_stop(struct devfreq *devfreq); extern void devfreq_monitor_suspend(struct devfreq *devfreq); @@ -38,4 +37,5 @@ extern int devfreq_add_governor(struct devfreq_governor *governor); extern int devfreq_remove_governor(struct devfreq_governor *governor); +extern int devfreq_get_freq_level(struct devfreq *devfreq, unsigned long freq); #endif /* _GOVERNOR_H */ diff -ruw linux-4.4.115/drivers/devfreq/governor_performance.c linux-4.4.115-fbx/drivers/devfreq/governor_performance.c --- linux-4.4.115/drivers/devfreq/governor_performance.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/governor_performance.c 2019-01-22 16:16:23.135243074 +0100 @@ -14,7 +14,8 @@ #include "governor.h" static int devfreq_performance_func(struct devfreq *df, - unsigned long *freq) + unsigned long *freq, + u32 *flag) { /* * target callback should be able to get floor value as @@ -31,13 +32,26 @@ unsigned int event, void *data) { int ret = 0; + unsigned long freq; - if (event == DEVFREQ_GOV_START) { mutex_lock(&devfreq->lock); + freq = devfreq->previous_freq; + switch (event) { + case DEVFREQ_GOV_START: + devfreq->profile->target(devfreq->dev.parent, + &freq, + DEVFREQ_FLAG_WAKEUP_MAXFREQ); + /* fall through */ + case DEVFREQ_GOV_RESUME: ret = update_devfreq(devfreq); - mutex_unlock(&devfreq->lock); + break; + case DEVFREQ_GOV_SUSPEND: + devfreq->profile->target(devfreq->dev.parent, + &freq, + DEVFREQ_FLAG_WAKEUP_MAXFREQ); + break; } - + mutex_unlock(&devfreq->lock); return ret; } diff -ruw linux-4.4.115/drivers/devfreq/governor_powersave.c linux-4.4.115-fbx/drivers/devfreq/governor_powersave.c --- linux-4.4.115/drivers/devfreq/governor_powersave.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/governor_powersave.c 2019-01-22 16:16:23.135243074 +0100 @@ -14,7 +14,8 @@ #include "governor.h" static int devfreq_powersave_func(struct devfreq *df, - unsigned long *freq) + unsigned long *freq, + u32 *flag) { /* * target callback should be able to get ceiling value as @@ -29,7 +30,7 @@ { int ret = 0; - if (event == DEVFREQ_GOV_START) { + if (event == DEVFREQ_GOV_START || event == DEVFREQ_GOV_RESUME) { mutex_lock(&devfreq->lock); ret = update_devfreq(devfreq); mutex_unlock(&devfreq->lock); diff -ruw linux-4.4.115/drivers/devfreq/governor_simpleondemand.c linux-4.4.115-fbx/drivers/devfreq/governor_simpleondemand.c --- linux-4.4.115/drivers/devfreq/governor_simpleondemand.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/governor_simpleondemand.c 2019-01-22 16:16:23.135243074 +0100 @@ -19,7 +19,8 @@ #define DFSO_UPTHRESHOLD (90) #define DFSO_DOWNDIFFERENCTIAL (5) static int devfreq_simple_ondemand_func(struct devfreq *df, - unsigned long *freq) + unsigned long *freq, + u32 *flag) { int err; struct devfreq_dev_status *stat; @@ -28,6 +29,7 @@ unsigned int dfso_downdifferential = DFSO_DOWNDIFFERENCTIAL; struct devfreq_simple_ondemand_data *data = df->data; unsigned long max = (df->max_freq) ? df->max_freq : UINT_MAX; + unsigned long min = (df->min_freq) ? df->min_freq : 0; err = devfreq_update_stats(df); if (err) @@ -45,18 +47,31 @@ dfso_upthreshold < dfso_downdifferential) return -EINVAL; - /* Assume MAX if it is going to be divided by zero */ - if (stat->total_time == 0) { - *freq = max; - return 0; - } - /* Prevent overflow */ if (stat->busy_time >= (1 << 24) || stat->total_time >= (1 << 24)) { stat->busy_time >>= 7; stat->total_time >>= 7; } + if (data && data->simple_scaling) { + if (stat->busy_time * 100 > + stat->total_time * dfso_upthreshold) + *freq = max; + else if (stat->busy_time * 100 < + stat->total_time * + (dfso_upthreshold - dfso_downdifferential)) + *freq = min; + else + *freq = df->previous_freq; + return 0; + } + + /* Assume MAX if it is going to be divided by zero */ + if (stat->total_time == 0) { + *freq = max; + return 0; + } + /* Set MAX if it's busy enough */ if (stat->busy_time * 100 > stat->total_time * dfso_upthreshold) { diff -ruw linux-4.4.115/drivers/devfreq/governor_userspace.c linux-4.4.115-fbx/drivers/devfreq/governor_userspace.c --- linux-4.4.115/drivers/devfreq/governor_userspace.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/governor_userspace.c 2019-01-22 16:16:23.135243074 +0100 @@ -22,7 +22,8 @@ bool valid; }; -static int devfreq_userspace_func(struct devfreq *df, unsigned long *freq) +static int devfreq_userspace_func(struct devfreq *df, unsigned long *freq, + u32 *flag) { struct userspace_data *data = df->data; diff -ruw linux-4.4.115/drivers/devfreq/Kconfig linux-4.4.115-fbx/drivers/devfreq/Kconfig --- linux-4.4.115/drivers/devfreq/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/Kconfig 2019-01-22 16:16:23.131243038 +0100 @@ -64,6 +64,114 @@ Otherwise, the governor does not change the frequnecy given at the initialization. +config DEVFREQ_GOV_QCOM_ADRENO_TZ + tristate "Qualcom Adreno Trustzone" + depends on QCOM_KGSL && QCOM_SCM + help + Trustzone based governor for the Adreno GPU. + Sets the frequency using a "on-demand" algorithm. + This governor is unlikely to be useful for other devices. + +config DEVFREQ_GOV_CPUFREQ + tristate "CPUfreq" + depends on CPU_FREQ + help + Chooses frequency based on the online CPUs' current frequency and a + CPU frequency to device frequency mapping table(s). This governor + can be useful for controlling devices such as DDR, cache, CCI, etc. + +config QCOM_BIMC_BWMON + tristate "QCOM BIMC Bandwidth monitor hardware" + depends on ARCH_QCOM + help + The BIMC Bandwidth monitor hardware allows for monitoring the + traffic coming from each master port connected to the BIMC. It also + has the capability to raise an IRQ when the count exceeds a + programmable limit. + +config DEVFREQ_GOV_QCOM_GPUBW_MON + tristate "GPU BW voting governor" + depends on DEVFREQ_GOV_QCOM_ADRENO_TZ + help + QTI GPU governor for GPU bus bandwidth voting. + This governor works together with QTI Adreno Trustzone governor, + and select bus frequency votes using an "on-demand" alorithm. + This governor is unlikely to be useful for non-QTI devices. + +config ARMBW_HWMON + tristate "ARM PMU Bandwidth monitor hardware" + depends on ARCH_QCOM + help + The PMU present on these ARM cores allow for the use of counters to + monitor the traffic coming from each core to the bus. It also has the + capability to raise an IRQ when the counter overflows, which can be + used to get an IRQ when the count exceeds a certain value + +config ARM_MEMLAT_MON + tristate "ARM CPU Memory Latency monitor hardware" + depends on ARCH_QCOM + help + The PMU present on these ARM cores allow for the use of counters to + monitor the memory latency characteristics of an ARM CPU workload. + This driver uses these counters to implement the APIs needed by + the mem_latency devfreq governor. + +config QCOMCCI_HWMON + tristate "QCOM CCI Cache monitor hardware" + depends on ARCH_QCOM + help + QCOM CCI has additional PMU counters that can be used to monitor + cache requests. QCOM CCI hardware monitor device configures these + registers to monitor cache and inform governor. It can also set an + IRQ when count exceeds a programmable limit. + +config QCOM_M4M_HWMON + tristate "QCOM M4M cache monitor hardware" + depends on ARCH_QCOM + help + QCOM M4M has counters that can be used to monitor requests coming to + M4M. QCOM M4M hardware monitor device programs corresponding registers + to monitor cache and inform governor. It can also set an IRQ when + count exceeds a programmable limit. + +config DEVFREQ_GOV_QCOM_BW_HWMON + tristate "HW monitor based governor for device BW" + depends on QCOM_BIMC_BWMON + help + HW monitor based governor for device to DDR bandwidth voting. + This governor sets the CPU BW vote by using BIMC counters to monitor + the CPU's use of DDR. Since this uses target specific counters it + can conflict with existing profiling tools. This governor is unlikely + to be useful for non-QCOM devices. + +config DEVFREQ_GOV_QCOM_CACHE_HWMON + tristate "HW monitor based governor for cache frequency" + help + HW monitor based governor for cache frequency scaling. This + governor sets the cache frequency by using PM counters to monitor the + CPU's use of cache. Since this governor uses some of the PM counters + it can conflict with existing profiling tools. This governor is + unlikely to be useful for other devices. + +config DEVFREQ_GOV_SPDM_HYP + bool "QCOM SPDM Hypervisor Governor" + depends on ARCH_QCOM + help + Hypervisor based governor for CPU bandwidth voting + for QCOM chipsets. + Sets the frequency using a "on-demand" algorithm. + This governor is unlikely to be useful for other devices. + +config DEVFREQ_GOV_MEMLAT + tristate "HW monitor based governor for device BW" + depends on ARM_MEMLAT_MON + help + HW monitor based governor for device to DDR bandwidth voting. + This governor sets the CPU BW vote based on stats obtained from memalat + monitor if it determines that a workload is memory latency bound. Since + this uses target specific counters it can conflict with existing profiling + tools. + comment "DEVFREQ Drivers" config ARM_EXYNOS4_BUS_DEVFREQ @@ -98,6 +206,48 @@ It reads ACTMON counters of memory controllers and adjusts the operating frequencies and voltages with OPP support. +config DEVFREQ_SIMPLE_DEV + tristate "Device driver for simple clock device with no status info" + select DEVFREQ_GOV_PERFORMANCE + select DEVFREQ_GOV_POWERSAVE + select DEVFREQ_GOV_USERSPACE + select DEVFREQ_GOV_CPUFREQ + help + Device driver for simple devices that control their frequency using + clock APIs and don't have any form of status reporting. + +config QCOM_DEVFREQ_DEVBW + bool "QCOM DEVFREQ device for device master <-> slave IB/AB BW voting" + depends on ARCH_QCOM + select DEVFREQ_GOV_PERFORMANCE + select DEVFREQ_GOV_POWERSAVE + select DEVFREQ_GOV_USERSPACE + select DEVFREQ_GOV_CPUFREQ + default n + help + Different devfreq governors use this devfreq device to make CPU to + DDR IB/AB bandwidth votes. This driver provides a SoC topology + agnostic interface to so that some of the devfreq governors can be + shared across SoCs. + +config SPDM_SCM + bool "QCOM SPDM SCM based call support" + depends on DEVFREQ_SPDM + help + SPDM driver support the dcvs algorithm logic being accessed via + scm or hvc calls. This adds the support for SPDM interaction to + tz via SCM based call. If not selected then Hypervior interaction + will be activated. + +config DEVFREQ_SPDM + bool "QCOM SPDM based bandwidth voting" + depends on ARCH_QCOM + select DEVFREQ_GOV_SPDM_HYP + help + This adds the support for SPDM based bandwidth voting on QCOM chipsets. + This driver allows any SPDM based client to vote for bandwidth. + Used with the QCOM SPDM Hypervisor Governor. + source "drivers/devfreq/event/Kconfig" endif # PM_DEVFREQ diff -ruw linux-4.4.115/drivers/devfreq/Makefile linux-4.4.115-fbx/drivers/devfreq/Makefile --- linux-4.4.115/drivers/devfreq/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/devfreq/Makefile 2019-01-22 16:16:23.131243038 +0100 @@ -4,11 +4,27 @@ obj-$(CONFIG_DEVFREQ_GOV_PERFORMANCE) += governor_performance.o obj-$(CONFIG_DEVFREQ_GOV_POWERSAVE) += governor_powersave.o obj-$(CONFIG_DEVFREQ_GOV_USERSPACE) += governor_userspace.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_ADRENO_TZ) += governor_msm_adreno_tz.o +obj-$(CONFIG_DEVFREQ_GOV_CPUFREQ) += governor_cpufreq.o +obj-$(CONFIG_QCOM_BIMC_BWMON) += bimc-bwmon.o +obj-$(CONFIG_ARMBW_HWMON) += armbw-pm.o +obj-$(CONFIG_ARM_MEMLAT_MON) += arm-memlat-mon.o +obj-$(CONFIG_QCOMCCI_HWMON) += msmcci-hwmon.o +obj-$(CONFIG_QCOM_M4M_HWMON) += m4m-hwmon.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_BW_HWMON) += governor_bw_hwmon.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_CACHE_HWMON) += governor_cache_hwmon.o +obj-$(CONFIG_DEVFREQ_GOV_SPDM_HYP) += governor_spdm_bw_hyp.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_GPUBW_MON) += governor_gpubw_mon.o +obj-$(CONFIG_DEVFREQ_GOV_QCOM_GPUBW_MON) += governor_bw_vbif.o +obj-$(CONFIG_DEVFREQ_GOV_MEMLAT) += governor_memlat.o # DEVFREQ Drivers obj-$(CONFIG_ARM_EXYNOS4_BUS_DEVFREQ) += exynos/ obj-$(CONFIG_ARM_EXYNOS5_BUS_DEVFREQ) += exynos/ obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra-devfreq.o +obj-$(CONFIG_QCOM_DEVFREQ_DEVBW) += devfreq_devbw.o +obj-$(CONFIG_DEVFREQ_SIMPLE_DEV) += devfreq_simple_dev.o +obj-$(CONFIG_DEVFREQ_SPDM) += devfreq_spdm.o devfreq_spdm_debugfs.o # DEVFREQ Event Drivers obj-$(CONFIG_PM_DEVFREQ_EVENT) += event/ diff -ruw linux-4.4.115/drivers/dma/Kconfig linux-4.4.115-fbx/drivers/dma/Kconfig --- linux-4.4.115/drivers/dma/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/dma/Kconfig 2019-01-22 16:16:23.139243110 +0100 @@ -417,6 +417,16 @@ Enable support for the QCOM BAM DMA controller. This controller provides DMA capabilities for a variety of on-chip devices. +config QCOM_SPS_DMA + tristate "Qualcomm technologies inc DMA driver for sps-BAM" + depends on ARCH_QCOM + select DMA_ENGINE + help + Enable support for Qualcomm technologies inc, BAM DMA engine. + This DMA-engine-driver is a wrapper of the sps-BAM library. DMA + engine callbacks are implemented using the sps-BAM functionality + to access HW. + config SIRF_DMA tristate "CSR SiRFprimaII/SiRFmarco DMA support" depends on ARCH_SIRF diff -ruw linux-4.4.115/drivers/dma/Makefile linux-4.4.115-fbx/drivers/dma/Makefile --- linux-4.4.115/drivers/dma/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/dma/Makefile 2019-01-22 16:16:23.139243110 +0100 @@ -65,5 +65,6 @@ obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o obj-$(CONFIG_ZX_DMA) += zx296702_dma.o +obj-$(CONFIG_QCOM_SPS_DMA) += qcom-sps-dma.o obj-y += xilinx/ diff -ruw linux-4.4.115/drivers/dma-buf/fence.c linux-4.4.115-fbx/drivers/dma-buf/fence.c --- linux-4.4.115/drivers/dma-buf/fence.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/dma-buf/fence.c 2019-01-22 16:16:23.139243110 +0100 @@ -304,8 +304,12 @@ spin_lock_irqsave(fence->lock, flags); ret = !list_empty(&cb->node); - if (ret) + if (ret) { list_del_init(&cb->node); + if (list_empty(&fence->cb_list)) + if (fence->ops->disable_signaling) + fence->ops->disable_signaling(fence); + } spin_unlock_irqrestore(fence->lock, flags); diff -ruw linux-4.4.115/drivers/edac/edac_core.h linux-4.4.115-fbx/drivers/edac/edac_core.h --- linux-4.4.115/drivers/edac/edac_core.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/edac/edac_core.h 2019-01-22 16:16:23.171243400 +0100 @@ -223,9 +223,11 @@ /* Per instance controls for this edac_device */ int log_ue; /* boolean for logging UEs */ int log_ce; /* boolean for logging CEs */ + int panic_on_ce; /* boolean for panic'ing on an CE */ int panic_on_ue; /* boolean for panic'ing on an UE */ unsigned poll_msec; /* number of milliseconds to poll interval */ unsigned long delay; /* number of jiffies for poll_msec */ + bool defer_work; /* Create a deferrable work for polling */ /* Additional top controller level attributes, but specified * by the low level driver. diff -ruw linux-4.4.115/drivers/edac/edac_device.c linux-4.4.115-fbx/drivers/edac/edac_device.c --- linux-4.4.115/drivers/edac/edac_device.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/edac/edac_device.c 2019-01-22 16:16:23.171243400 +0100 @@ -411,9 +411,15 @@ * to used in the time period calculation * then calc the number of jiffies that represents */ + if (!msec) + msec = 1000; edac_dev->poll_msec = msec; edac_dev->delay = msecs_to_jiffies(msec); + if (edac_dev->defer_work) + INIT_DEFERRABLE_WORK(&edac_dev->work, + edac_device_workq_function); + else INIT_DELAYED_WORK(&edac_dev->work, edac_device_workq_function); /* optimize here for the 1 second case, which will be normal value, to @@ -523,7 +529,7 @@ * enable workq processing on this instance, * default = 1000 msec */ - edac_device_workq_setup(edac_dev, 1000); + edac_device_workq_setup(edac_dev, edac_dev->poll_msec); } else { edac_dev->op_state = OP_RUNNING_INTERRUPT; } @@ -608,6 +614,12 @@ return edac_dev->log_ue; } +static inline int edac_device_get_panic_on_ce(struct edac_device_ctl_info + *edac_dev) +{ + return edac_dev->panic_on_ce; +} + static inline int edac_device_get_panic_on_ue(struct edac_device_ctl_info *edac_dev) { @@ -657,6 +669,11 @@ "CE: %s instance: %s block: %s '%s'\n", edac_dev->ctl_name, instance->name, block ? block->name : "N/A", msg); + + if (edac_device_get_panic_on_ce(edac_dev)) + panic("EDAC %s: CE instance: %s block %s '%s'\n", + edac_dev->ctl_name, instance->name, + block ? block->name : "N/A", msg); } EXPORT_SYMBOL_GPL(edac_device_handle_ce); diff -ruw linux-4.4.115/drivers/edac/edac_device_sysfs.c linux-4.4.115-fbx/drivers/edac/edac_device_sysfs.c --- linux-4.4.115/drivers/edac/edac_device_sysfs.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/edac/edac_device_sysfs.c 2019-01-22 16:16:23.171243400 +0100 @@ -62,6 +62,13 @@ return count; } +/* 'panic_on_ce' */ +static ssize_t edac_device_ctl_panic_on_ce_show(struct edac_device_ctl_info + *ctl_info, char *data) +{ + return snprintf(data, PAGE_SIZE, "%u\n", ctl_info->panic_on_ce); +} + /* 'panic_on_ue' */ static ssize_t edac_device_ctl_panic_on_ue_show(struct edac_device_ctl_info *ctl_info, char *data) @@ -69,6 +76,21 @@ return sprintf(data, "%u\n", ctl_info->panic_on_ue); } +static ssize_t edac_device_ctl_panic_on_ce_store(struct edac_device_ctl_info + *ctl_info, const char *data, + size_t count) +{ + unsigned long val; + + /* if parameter is zero, turn off flag, if non-zero turn on flag */ + if (kstrtoul(data, 0, &val) < 0) + return -EINVAL; + + ctl_info->panic_on_ce = !!val; + + return count; +} + static ssize_t edac_device_ctl_panic_on_ue_store(struct edac_device_ctl_info *ctl_info, const char *data, size_t count) @@ -156,6 +178,9 @@ edac_device_ctl_log_ue_show, edac_device_ctl_log_ue_store); CTL_INFO_ATTR(log_ce, S_IRUGO | S_IWUSR, edac_device_ctl_log_ce_show, edac_device_ctl_log_ce_store); +CTL_INFO_ATTR(panic_on_ce, S_IRUGO | S_IWUSR, + edac_device_ctl_panic_on_ce_show, + edac_device_ctl_panic_on_ce_store); CTL_INFO_ATTR(panic_on_ue, S_IRUGO | S_IWUSR, edac_device_ctl_panic_on_ue_show, edac_device_ctl_panic_on_ue_store); @@ -164,6 +189,7 @@ /* Base Attributes of the EDAC_DEVICE ECC object */ static struct ctl_info_attribute *device_ctrl_attr[] = { + &attr_ctl_info_panic_on_ce, &attr_ctl_info_panic_on_ue, &attr_ctl_info_log_ue, &attr_ctl_info_log_ce, diff -ruw linux-4.4.115/drivers/edac/Kconfig linux-4.4.115-fbx/drivers/edac/Kconfig --- linux-4.4.115/drivers/edac/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/edac/Kconfig 2019-01-22 16:16:23.167243364 +0100 @@ -390,4 +390,44 @@ Support for error detection and correction on the APM X-Gene family of SOCs. +config EDAC_CORTEX_ARM64 + depends on EDAC_MM_EDAC && ARM64 + bool "ARM Cortex A CPUs L1/L2 Caches" + help + Support for error detection and correction on the + ARM Cortex A53 and A57 CPUs. For debugging issues having to do with + stability and overall system health, you should probably say 'Y' + here. + +config EDAC_CORTEX_ARM64_PANIC_ON_CE + depends on EDAC_CORTEX_ARM64 + bool "Panic on correctable errors" + help + Forcibly cause a kernel panic if an correctable error (CE) is + detected, even though the error is (by definition) correctable and + would otherwise result in no adverse system effects. This can reduce + debugging times on hardware which may be operating at voltages or + frequencies outside normal specification. + + For production builds, you should definitely say 'N' here. + +config EDAC_CORTEX_ARM64_DBE_IRQ_ONLY + depends on EDAC_CORTEX_ARM64 + bool "Only check for parity errors when an irq is generated" + help + In ARM64, parity errors will cause an interrupt + to be triggered but may also cause a data abort to + occur. Only check for EDAC errors for the interrupt. + If unsure, say no. + +config EDAC_CORTEX_ARM64_PANIC_ON_UE + depends on EDAC_CORTEX_ARM64 + bool "Panic on uncorrectable errors" + help + Forcibly cause a kernel panic if an uncorrectable error (UE) is + detected. This can reduce debugging times on hardware which may be + operating at voltages or frequencies outside normal specification. + + For production builds, you should probably say 'N' here. + endif # EDAC diff -ruw linux-4.4.115/drivers/edac/Makefile linux-4.4.115-fbx/drivers/edac/Makefile --- linux-4.4.115/drivers/edac/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/edac/Makefile 2019-01-22 16:16:23.167243364 +0100 @@ -70,3 +70,4 @@ obj-$(CONFIG_EDAC_ALTERA_MC) += altera_edac.o obj-$(CONFIG_EDAC_SYNOPSYS) += synopsys_edac.o obj-$(CONFIG_EDAC_XGENE) += xgene_edac.o +obj-$(CONFIG_EDAC_CORTEX_ARM64) += cortex_arm64_edac.o diff -ruw linux-4.4.115/drivers/extcon/extcon.c linux-4.4.115-fbx/drivers/extcon/extcon.c --- linux-4.4.115/drivers/extcon/extcon.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/extcon/extcon.c 2019-01-22 16:16:23.183243509 +0100 @@ -63,6 +63,12 @@ [EXTCON_JACK_SPDIF_IN] = "SPDIF-IN", [EXTCON_JACK_SPDIF_OUT] = "SPDIF-OUT", + /* connector orientation 0 - CC1, 1 - CC2 */ + [EXTCON_USB_CC] = "USB-CC", + + /* connector speed 0 - High Speed, 1 - Super Speed */ + [EXTCON_USB_SPEED] = "USB-SPEED", + /* Display external connector */ [EXTCON_DISP_HDMI] = "HDMI", [EXTCON_DISP_MHL] = "MHL", @@ -78,7 +84,7 @@ }; static struct class *extcon_class; -#if defined(CONFIG_ANDROID) +#if defined(CONFIG_ANDROID) && !IS_ENABLED(CONFIG_SWITCH) static struct class_compat *switch_class; #endif /* CONFIG_ANDROID */ @@ -635,7 +641,7 @@ return PTR_ERR(extcon_class); extcon_class->dev_groups = extcon_groups; -#if defined(CONFIG_ANDROID) +#if defined(CONFIG_ANDROID) && !IS_ENABLED(CONFIG_SWITCH) switch_class = class_compat_register("switch"); if (WARN(!switch_class, "cannot allocate")) return -ENOMEM; @@ -921,7 +927,7 @@ put_device(&edev->dev); goto err_dev; } -#if defined(CONFIG_ANDROID) +#if defined(CONFIG_ANDROID) && !IS_ENABLED(CONFIG_SWITCH) if (switch_class) ret = class_compat_create_link(switch_class, &edev->dev, NULL); #endif /* CONFIG_ANDROID */ @@ -1010,7 +1016,7 @@ kfree(edev->cables); } -#if defined(CONFIG_ANDROID) +#if defined(CONFIG_ANDROID) && !IS_ENABLED(CONFIG_SWITCH) if (switch_class) class_compat_remove_link(switch_class, &edev->dev, NULL); #endif @@ -1139,7 +1145,7 @@ static void __exit extcon_class_exit(void) { -#if defined(CONFIG_ANDROID) +#if defined(CONFIG_ANDROID) && !IS_ENABLED(CONFIG_SWITCH) class_compat_unregister(switch_class); #endif class_destroy(extcon_class); diff -ruw linux-4.4.115/drivers/firmware/efi/libstub/Makefile linux-4.4.115-fbx/drivers/firmware/efi/libstub/Makefile --- linux-4.4.115/drivers/firmware/efi/libstub/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/firmware/efi/libstub/Makefile 2019-01-22 16:16:23.191243581 +0100 @@ -8,20 +8,24 @@ cflags-$(CONFIG_X86_64) := -mcmodel=small cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2 \ -fPIC -fno-strict-aliasing -mno-red-zone \ - -mno-mmx -mno-sse -DDISABLE_BRANCH_PROFILING + -mno-mmx -mno-sse -cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) +cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -fpie cflags-$(CONFIG_ARM) := $(subst -pg,,$(KBUILD_CFLAGS)) \ -fno-builtin -fpic -mno-single-pic-base cflags-$(CONFIG_EFI_ARMSTUB) += -I$(srctree)/scripts/dtc/libfdt -KBUILD_CFLAGS := $(cflags-y) \ +KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-ffreestanding) \ $(call cc-option,-fno-stack-protector) GCOV_PROFILE := n KASAN_SANITIZE := n +UBSAN_SANITIZE := n + +# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. +KCOV_INSTRUMENT := n lib-y := efi-stub-helper.o @@ -34,7 +38,8 @@ lib-$(CONFIG_EFI_ARMSTUB) += arm-stub.o fdt.o string.o \ $(patsubst %.c,lib-%.o,$(arm-deps)) -lib-$(CONFIG_ARM64) += arm64-stub.o +lib-$(CONFIG_ARM) += arm32-stub.o +lib-$(CONFIG_ARM64) += arm64-stub.o random.o CFLAGS_arm64-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) # diff -ruw linux-4.4.115/drivers/firmware/efi/Makefile linux-4.4.115-fbx/drivers/firmware/efi/Makefile --- linux-4.4.115/drivers/firmware/efi/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/firmware/efi/Makefile 2019-01-22 16:16:23.191243581 +0100 @@ -18,3 +18,6 @@ obj-$(CONFIG_EFI_RUNTIME_WRAPPERS) += runtime-wrappers.o obj-$(CONFIG_EFI_STUB) += libstub/ obj-$(CONFIG_EFI_FAKE_MEMMAP) += fake_mem.o + +arm-obj-$(CONFIG_EFI) := arm-init.o arm-runtime.o +obj-$(CONFIG_ARM64) += $(arm-obj-y) diff -ruw linux-4.4.115/drivers/firmware/Kconfig linux-4.4.115-fbx/drivers/firmware/Kconfig --- linux-4.4.115/drivers/firmware/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/firmware/Kconfig 2019-10-29 09:26:23.537202141 +0100 @@ -173,8 +173,12 @@ def_bool y depends on QCOM_SCM && ARM64 +config HAVE_ARM_SMCCC + bool + source "drivers/firmware/broadcom/Kconfig" source "drivers/firmware/google/Kconfig" source "drivers/firmware/efi/Kconfig" +source "drivers/firmware/qcom/Kconfig" endmenu diff -ruw linux-4.4.115/drivers/firmware/Makefile linux-4.4.115-fbx/drivers/firmware/Makefile --- linux-4.4.115/drivers/firmware/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/firmware/Makefile 2019-01-22 16:16:23.187243545 +0100 @@ -23,3 +23,4 @@ obj-$(CONFIG_GOOGLE_FIRMWARE) += google/ obj-$(CONFIG_EFI) += efi/ obj-$(CONFIG_UEFI_CPER) += efi/ +obj-$(CONFIG_MSM_TZ_LOG) += qcom/ \ No newline at end of file diff -ruw linux-4.4.115/drivers/firmware/psci.c linux-4.4.115-fbx/drivers/firmware/psci.c --- linux-4.4.115/drivers/firmware/psci.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/firmware/psci.c 2019-01-22 16:16:23.195243617 +0100 @@ -13,6 +13,8 @@ #define pr_fmt(fmt) "psci: " fmt +#include +#include #include #include #include @@ -20,10 +22,12 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -58,8 +62,6 @@ typedef unsigned long (psci_fn)(unsigned long, unsigned long, unsigned long, unsigned long); -asmlinkage psci_fn __invoke_psci_fn_hvc; -asmlinkage psci_fn __invoke_psci_fn_smc; static psci_fn *invoke_psci_fn; enum psci_function { @@ -225,6 +227,130 @@ psci_func_id, 0, 0); } +#ifdef CONFIG_CPU_IDLE +static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state); + +#ifdef CONFIG_ARM_PSCI +static int psci_cpu_init(struct device_node *cpu_node, int cpu) +{ + return 0; +} +#endif + +static int psci_dt_cpu_init_idle(struct device_node *cpu_node, int cpu) +{ + int i, ret, count = 0; + u32 *psci_states; + struct device_node *state_node; + + /* + * If the PSCI cpu_suspend function hook has not been initialized + * idle states must not be enabled, so bail out + */ + if (!psci_ops.cpu_suspend) + return -EOPNOTSUPP; + + /* Count idle states */ + while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states", + count))) { + count++; + of_node_put(state_node); + } + + if (!count) + return -ENODEV; + + psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL); + if (!psci_states) + return -ENOMEM; + + for (i = 0; i < count; i++) { + u32 state; + + state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i); + + ret = of_property_read_u32(state_node, + "arm,psci-suspend-param", + &state); + if (ret) { + pr_warn(" * %s missing arm,psci-suspend-param property\n", + state_node->full_name); + of_node_put(state_node); + goto free_mem; + } + + of_node_put(state_node); + pr_debug("psci-power-state %#x index %d\n", state, i); + if (!psci_power_state_is_valid(state)) { + pr_warn("Invalid PSCI power state %#x\n", state); + ret = -EINVAL; + goto free_mem; + } + psci_states[i] = state; + } + /* Idle states parsed correctly, initialize per-cpu pointer */ + per_cpu(psci_power_state, cpu) = psci_states; + return 0; + +free_mem: + kfree(psci_states); + return ret; +} + +int psci_cpu_init_idle(unsigned int cpu) +{ + struct device_node *cpu_node; + int ret; + + cpu_node = of_get_cpu_node(cpu, NULL); + if (!cpu_node) + return -ENODEV; + + ret = psci_dt_cpu_init_idle(cpu_node, cpu); + + of_node_put(cpu_node); + + return ret; +} + +static int psci_suspend_finisher(unsigned long state_id) +{ + return psci_ops.cpu_suspend(state_id, virt_to_phys(cpu_resume)); +} + +int psci_cpu_suspend_enter(unsigned long state_id) +{ + int ret; + /* + * idle state_id 0 corresponds to wfi, should never be called + * from the cpu_suspend operations + */ + if (WARN_ON_ONCE(!state_id)) + return -EINVAL; + + if (!psci_power_state_loses_context(state_id)) + ret = psci_ops.cpu_suspend(state_id, 0); + else + ret = cpu_suspend(state_id, psci_suspend_finisher); + + return ret; +} + +/* ARM specific CPU idle operations */ +#ifdef CONFIG_ARM +static struct cpuidle_ops psci_cpuidle_ops __initdata = { + .suspend = psci_cpu_suspend_enter, +#ifdef CONFIG_ARM_PSCI + .init = psci_cpu_init, +#else + .init = psci_dt_cpu_init_idle, +#endif +}; + +CPUIDLE_METHOD_OF_DECLARE(psci, "psci", &psci_cpuidle_ops); +#endif +#endif + static int psci_system_suspend(unsigned long unused) { return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND), @@ -305,6 +431,8 @@ static void __init psci_0_2_set_functions(void) { pr_info("Using standard PSCI v0.2 function IDs\n"); + psci_ops.get_version = psci_get_version; + psci_function_id[PSCI_FN_CPU_SUSPEND] = PSCI_FN_NATIVE(0_2, CPU_SUSPEND); psci_ops.cpu_suspend = psci_cpu_suspend; diff -ruw linux-4.4.115/drivers/gpio/gpiolib-acpi.c linux-4.4.115-fbx/drivers/gpio/gpiolib-acpi.c --- linux-4.4.115/drivers/gpio/gpiolib-acpi.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpio/gpiolib-acpi.c 2019-01-22 16:16:23.211243762 +0100 @@ -597,7 +597,7 @@ int idx, i; for (i = 0, idx = 0; idx <= index; i++) { - struct acpi_gpio_info info; + struct acpi_gpio_info info = {0, 0}; struct gpio_desc *desc; desc = acpi_get_gpiod_by_index(adev, NULL, i, &info); diff -ruw linux-4.4.115/drivers/gpio/gpio-pca953x.c linux-4.4.115-fbx/drivers/gpio/gpio-pca953x.c --- linux-4.4.115/drivers/gpio/gpio-pca953x.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpio/gpio-pca953x.c 2019-01-22 16:16:23.207243726 +0100 @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -681,8 +682,16 @@ invert = pdata->invert; chip->names = pdata->names; } else { + struct gpio_desc *reset_gpio; + chip->gpio_start = -1; irq_base = 0; + + /* See if we need to de-assert a reset pin */ + reset_gpio = devm_gpiod_get_optional(&client->dev, "reset", + GPIOD_OUT_LOW); + if (IS_ERR(reset_gpio)) + return PTR_ERR(reset_gpio); } chip->client = client; diff -ruw linux-4.4.115/drivers/gpio/Kconfig linux-4.4.115-fbx/drivers/gpio/Kconfig --- linux-4.4.115/drivers/gpio/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpio/Kconfig 2019-10-29 09:26:23.541202180 +0100 @@ -338,6 +338,25 @@ help Say yes here to support the PXA GPIO device +config GPIO_QPNP_PIN + tristate "Qualcomm Technologies, Inc. QPNP GPIO support" + depends on SPMI + help + Say 'y' here to include support for the Qualcomm Technologies, Inc. + QPNP GPIO driver. This driver supports Device Tree and allows a + device_node to be registered as a gpio-controller. It does not handle + GPIO interrupts directly; they are handled via the SPMI arbiter + interrupt driver. + +config GPIO_QPNP_PIN_DEBUG + bool "Qualcomm Technologies, Inc. QPNP GPIO debug support" + depends on GPIO_QPNP_PIN && DEBUG_FS + help + Say 'y' here to include debug support for the Qualcomm Technologies, + Inc. QPNP GPIO driver. This provides a userspace debug interface to + get and set all of the supported features of PMIC GPIO and MPP pins + including those which are managed by the gpio framework. + config GPIO_RCAR tristate "Renesas R-Car GPIO" depends on ARCH_SHMOBILE || COMPILE_TEST diff -ruw linux-4.4.115/drivers/gpio/Makefile linux-4.4.115-fbx/drivers/gpio/Makefile --- linux-4.4.115/drivers/gpio/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpio/Makefile 2019-04-24 19:28:47.284498086 +0200 @@ -76,6 +76,7 @@ obj-$(CONFIG_GPIO_PCH) += gpio-pch.o obj-$(CONFIG_GPIO_PL061) += gpio-pl061.o obj-$(CONFIG_GPIO_PXA) += gpio-pxa.o +obj-$(CONFIG_GPIO_QPNP_PIN) += qpnp-pin.o obj-$(CONFIG_GPIO_RC5T583) += gpio-rc5t583.o obj-$(CONFIG_GPIO_RDC321X) += gpio-rdc321x.o obj-$(CONFIG_GPIO_RCAR) += gpio-rcar.o @@ -119,3 +120,4 @@ obj-$(CONFIG_GPIO_ZEVIO) += gpio-zevio.o obj-$(CONFIG_GPIO_ZYNQ) += gpio-zynq.o obj-$(CONFIG_GPIO_ZX) += gpio-zx.o +obj-$(CONFIG_MSM_SMP2P) += gpio-msm-smp2p.o diff -ruw linux-4.4.115/drivers/gpu/drm/drm_edid.c linux-4.4.115-fbx/drivers/gpu/drm/drm_edid.c --- linux-4.4.115/drivers/gpu/drm/drm_edid.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/drm_edid.c 2019-10-29 09:26:23.601202767 +0100 @@ -91,6 +91,14 @@ #define LEVEL_GTF2 2 #define LEVEL_CVT 3 +/*Enum storing luminance types for HDR blocks in EDID*/ +enum luminance_value { + NO_LUMINANCE_DATA = 3, + MAXIMUM_LUMINANCE = 4, + FRAME_AVERAGE_LUMINANCE = 5, + MINIMUM_LUMINANCE = 6 +}; + static struct edid_quirk { char vendor[4]; int product_id; @@ -994,9 +1002,224 @@ .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, /* 64 - 1920x1080@100Hz */ { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448, - 2492, 2640, 0, 1080, 1084, 1094, 1125, 0, + 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 65 - 1280x720@24Hz */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 59400, 1280, 3040, + 3080, 3300, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 66 - 1280x720@25Hz */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3700, + 3740, 3960, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 67 - 1280x720@30Hz */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3040, + 3080, 3300, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 68 - 1280x720@50Hz */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720, + 1760, 1980, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 69 - 1280x720@60Hz */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390, + 1430, 1650, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 70 - 1280x720@100Hz */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1720, + 1760, 1980, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 71 - 1280x720@120Hz */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1390, + 1430, 1650, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 72 - 1920x1080@24Hz */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2558, + 2602, 2750, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 73 - 1920x1080@25Hz */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448, + 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 74 - 1920x1080@30Hz */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008, + 2052, 2200, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 75 - 1920x1080@50Hz */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448, + 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 76 - 1920x1080@60Hz */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008, + 2052, 2200, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 77 - 1920x1080@100Hz */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448, + 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 78 - 1920x1080@120Hz */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2008, + 2052, 2200, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 79 - 1680x720@24Hz */ + { DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 3040, + 3080, 3300, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 80 - 1680x720@25Hz */ + { DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2908, + 2948, 3168, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 81 - 1680x720@30Hz */ + { DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2380, + 2420, 2640, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 82 - 1680x720@50Hz */ + { DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 82500, 1680, 1940, + 1980, 2200, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 83 - 1680x720@60Hz */ + { DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 99000, 1680, 1940, + 1980, 2200, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 84 - 1680x720@100Hz */ + { DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 165000, 1680, 1740, + 1780, 2000, 0, 720, 725, 730, 825, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 85 - 1680x720@120Hz */ + { DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 198000, 1680, 1740, + 1780, 2000, 0, 720, 725, 730, 825, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 86 - 2560x1080@24Hz */ + { DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 99000, 2560, 3558, + 3602, 3750, 0, 1080, 1084, 1089, 1100, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 87 - 2560x1080@25Hz */ + { DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 90000, 2560, 3008, + 3052, 3200, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 88 - 2560x1080@30Hz */ + { DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 118800, 2560, 3328, + 3372, 3520, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 89 - 2560x1080@50Hz */ + { DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 185625, 2560, 3108, + 3152, 3300, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 90 - 2560x1080@60Hz */ + { DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 198000, 2560, 2808, + 2852, 3000, 0, 1080, 1084, 1089, 1100, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 91 - 2560x1080@100Hz */ + { DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 371250, 2560, 2778, + 2822, 2970, 0, 1080, 1084, 1089, 1250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 92 - 2560x1080@120Hz */ + { DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 495000, 2560, 3108, + 3152, 3300, 0, 1080, 1084, 1089, 1250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 93 - 3840x2160p@24Hz 16:9 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116, + 5204, 5500, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 94 - 3840x2160p@25Hz 16:9 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896, + 4984, 5280, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 95 - 3840x2160p@30Hz 16:9 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016, + 4104, 4400, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 96 - 3840x2160p@50Hz 16:9 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896, + 4984, 5280, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 97 - 3840x2160p@60Hz 16:9 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016, + 4104, 4400, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 98 - 4096x2160p@24Hz 256:135 */ + { DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5116, + 5204, 5500, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, }, + /* 99 - 4096x2160p@25Hz 256:135 */ + { DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5064, + 5152, 5280, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, }, + /* 100 - 4096x2160p@30Hz 256:135 */ + { DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 4184, + 4272, 4400, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, }, + /* 101 - 4096x2160p@50Hz 256:135 */ + { DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 5064, + 5152, 5280, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, }, + /* 102 - 4096x2160p@60Hz 256:135 */ + { DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 4184, + 4272, 4400, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, }, + /* 103 - 3840x2160p@24Hz 64:27 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116, + 5204, 5500, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 104 - 3840x2160p@25Hz 64:27 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896, + 4984, 5280, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 105 - 3840x2160p@30Hz 64:27 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016, + 4104, 4400, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 106 - 3840x2160p@50Hz 64:27 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896, + 4984, 5280, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, + /* 107 - 3840x2160p@60Hz 64:27 */ + { DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016, + 4104, 4400, 0, 2160, 2168, 2178, 2250, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, }, }; /* @@ -1275,6 +1498,10 @@ * level, drivers must make all reasonable efforts to expose it as an I2C * adapter and use drm_get_edid() instead of abusing this function. * + * The EDID may be overridden using debugfs override_edid or firmare EDID + * (drm_load_edid_firmware() and drm.edid_firmware parameter), in this priority + * order. Having either of them bypasses actual EDID reads. + * * Return: Pointer to valid EDID or NULL if we couldn't find any. */ struct edid *drm_do_get_edid(struct drm_connector *connector, @@ -1285,6 +1512,19 @@ int i, j = 0, valid_extensions = 0; u8 *block, *new; bool print_bad_edid = !connector->bad_edid_counter || (drm_debug & DRM_UT_KMS); + struct edid *override = NULL; + + if (connector->override_edid) + override = drm_edid_duplicate((const struct edid *) + connector->edid_blob_ptr->data); + +#ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE + if (!override) + override = drm_load_edid_firmware(connector); +#endif + + if (!IS_ERR_OR_NULL(override)) + return override; if ((block = kmalloc(EDID_LENGTH, GFP_KERNEL)) == NULL) return NULL; @@ -2487,12 +2727,16 @@ return closure.modes; } - +#define VIDEO_CAPABILITY_EXTENDED_DATA_BLOCK 0x0 #define AUDIO_BLOCK 0x01 #define VIDEO_BLOCK 0x02 #define VENDOR_BLOCK 0x03 #define SPEAKER_BLOCK 0x04 +#define HDR_STATIC_METADATA_EXTENDED_DATA_BLOCK 0x06 +#define COLORIMETRY_EXTENDED_DATA_BLOCK 0x05 +#define EXTENDED_TAG 0x07 #define VIDEO_CAPABILITY_BLOCK 0x07 +#define Y420_VIDEO_DATA_BLOCK 0x0E #define EDID_BASIC_AUDIO (1 << 6) #define EDID_CEA_YCRCB444 (1 << 5) #define EDID_CEA_YCRCB422 (1 << 4) @@ -3081,6 +3325,21 @@ return hdmi_id == HDMI_IEEE_OUI; } +static bool cea_db_is_hdmi_hf_vsdb(const u8 *db) +{ + int hdmi_id; + + if (cea_db_tag(db) != VENDOR_BLOCK) + return false; + + if (cea_db_payload_len(db) < 7) + return false; + + hdmi_id = db[1] | (db[2] << 8) | (db[3] << 16); + + return hdmi_id == HDMI_IEEE_OUI_HF; +} + #define for_each_cea_db(cea, i, start, end) \ for ((i) = (start); (i) < (end) && (i) + cea_db_payload_len(&(cea)[(i)]) < (end); (i) += cea_db_payload_len(&(cea)[(i)]) + 1) @@ -3204,6 +3463,311 @@ } static void +parse_hdmi_hf_vsdb(struct drm_connector *connector, const u8 *db) +{ + u8 len = cea_db_payload_len(db); + + if (len < 7) + return; + + if (db[4] != 1) + return; /* invalid version */ + + connector->max_tmds_char = db[5] * 5; + connector->scdc_present = db[6] & (1 << 7); + connector->rr_capable = db[6] & (1 << 6); + connector->flags_3d = db[6] & 0x7; + connector->supports_scramble = connector->scdc_present && + (db[6] & (1 << 3)); + + DRM_DEBUG_KMS("HDMI v2: max TMDS char %d, " + "scdc %s, " + "rr %s, " + "3D flags 0x%x, " + "scramble %s\n", + connector->max_tmds_char, + connector->scdc_present ? "available" : "not available", + connector->rr_capable ? "capable" : "not capable", + connector->flags_3d, + connector->supports_scramble ? + "supported" : "not supported"); +} + +static void +drm_hdmi_extract_vsdbs_info(struct drm_connector *connector, struct edid *edid) +{ + const u8 *cea = drm_find_cea_extension(edid); + const u8 *db = NULL; + + if (cea && cea_revision(cea) >= 3) { + int i, start, end; + + if (cea_db_offsets(cea, &start, &end)) + return; + + for_each_cea_db(cea, i, start, end) { + db = &cea[i]; + + if (cea_db_tag(db) == VENDOR_BLOCK) { + /* HDMI Vendor-Specific Data Block */ + if (cea_db_is_hdmi_vsdb(db)) + parse_hdmi_vsdb(connector, db); + /* HDMI Forum Vendor-Specific Data Block */ + else if (cea_db_is_hdmi_hf_vsdb(db)) + parse_hdmi_hf_vsdb(connector, db); + } + } + } +} + +/* + * drm_extract_vcdb_info - Parse the HDMI Video Capability Data Block + * @connector: connector corresponding to the HDMI sink + * @db: start of the CEA vendor specific block + * + * Parses the HDMI VCDB to extract sink info for @connector. + */ +static void +drm_extract_vcdb_info(struct drm_connector *connector, const u8 *db) +{ + /* + * Check if the sink specifies underscan + * support for: + * BIT 5: preferred video format + * BIT 3: IT video format + * BIT 1: CE video format + */ + + connector->pt_scan_info = + (db[2] & (BIT(4) | BIT(5))) >> 4; + connector->it_scan_info = + (db[2] & (BIT(3) | BIT(2))) >> 2; + connector->ce_scan_info = + db[2] & (BIT(1) | BIT(0)); + connector->rgb_qs = + db[2] & BIT(6); + connector->yuv_qs = + db[2] & BIT(7); + + DRM_DEBUG_KMS("Scan Info (pt|it|ce): (%d|%d|%d)", + (int) connector->pt_scan_info, + (int) connector->it_scan_info, + (int) connector->ce_scan_info); + DRM_DEBUG_KMS("rgb_quant_range_select %d", connector->rgb_qs); + DRM_DEBUG_KMS("ycc_quant_range_select %d", connector->yuv_qs); +} + +static bool drm_edid_is_luminance_value_present( +u32 block_length, enum luminance_value value) +{ + return block_length > NO_LUMINANCE_DATA && value <= block_length; +} + +/* + * drm_extract_hdr_db - Parse the HDMI HDR extended block + * @connector: connector corresponding to the HDMI sink + * @db: start of the HDMI HDR extended block + * + * Parses the HDMI HDR extended block to extract sink info for @connector. + */ +static void +drm_extract_hdr_db(struct drm_connector *connector, const u8 *db) +{ + + u8 len = 0; + + if (!db) { + DRM_ERROR("invalid db\n"); + return; + } + + len = db[0] & 0x1f; + /* Byte 3: Electro-Optical Transfer Functions */ + connector->hdr_eotf = db[2] & 0x3F; + + /* Byte 4: Static Metadata Descriptor Type 1 */ + connector->hdr_metadata_type_one = (db[3] & BIT(0)); + + /* Byte 5: Desired Content Maximum Luminance */ + if (drm_edid_is_luminance_value_present(len, MAXIMUM_LUMINANCE)) + connector->hdr_max_luminance = + db[MAXIMUM_LUMINANCE]; + + /* Byte 6: Desired Content Max Frame-average Luminance */ + if (drm_edid_is_luminance_value_present(len, FRAME_AVERAGE_LUMINANCE)) + connector->hdr_avg_luminance = + db[FRAME_AVERAGE_LUMINANCE]; + + /* Byte 7: Desired Content Min Luminance */ + if (drm_edid_is_luminance_value_present(len, MINIMUM_LUMINANCE)) + connector->hdr_min_luminance = + db[MINIMUM_LUMINANCE]; + + connector->hdr_supported = true; + + DRM_DEBUG_KMS("HDR electro-optical %d\n", connector->hdr_eotf); + DRM_DEBUG_KMS("metadata desc 1 %d\n", connector->hdr_metadata_type_one); + DRM_DEBUG_KMS("max luminance %d\n", connector->hdr_max_luminance); + DRM_DEBUG_KMS("avg luminance %d\n", connector->hdr_avg_luminance); + DRM_DEBUG_KMS("min luminance %d\n", connector->hdr_min_luminance); +} + +/* + * drm_extract_colorimetry_db - Parse the HDMI colorimetry extended block + * @connector: connector corresponding to the HDMI sink + * @db: start of the HDMI colorimetry extended block + * + * Parses the HDMI colorimetry block to extract sink info for @connector. + */ +static void +drm_extract_clrmetry_db(struct drm_connector *connector, const u8 *db) +{ + + if (!db) { + DRM_ERROR("invalid db\n"); + return; + } + + /* Bit 0: xvYCC_601 */ + if (db[2] & BIT(0)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_xvYCC_601; + /* Bit 0: xvYCC_709 */ + if (db[2] & BIT(1)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_xvYCC_709; + /* Bit 0: sYCC_601 */ + if (db[2] & BIT(2)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_sYCC_601; + /* Bit 0: ADBYCC_601 */ + if (db[2] & BIT(3)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_ADBYCC_601; + /* Bit 0: ADB_RGB */ + if (db[2] & BIT(4)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_ADB_RGB; + /* Bit 0: BT2020_CYCC */ + if (db[2] & BIT(5)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_BT2020_CYCC; + /* Bit 0: BT2020_YCC */ + if (db[2] & BIT(6)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_BT2020_YCC; + /* Bit 0: BT2020_RGB */ + if (db[2] & BIT(7)) + connector->color_enc_fmt |= DRM_EDID_COLORIMETRY_BT2020_RGB; + + DRM_DEBUG_KMS("colorimetry fmt 0x%x\n", connector->color_enc_fmt); +} + +/* + * drm_hdmi_extract_extended_blk_info - Parse the HDMI extended tag blocks + * @connector: connector corresponding to the HDMI sink + * @edid: handle to the EDID structure + * Parses the all extended tag blocks extract sink info for @connector. + */ +static void +drm_hdmi_extract_extended_blk_info(struct drm_connector *connector, +struct edid *edid) +{ + const u8 *cea = drm_find_cea_extension(edid); + const u8 *db = NULL; + + if (cea && cea_revision(cea) >= 3) { + int i, start, end; + + if (cea_db_offsets(cea, &start, &end)) + return; + + for_each_cea_db(cea, i, start, end) { + db = &cea[i]; + + if (cea_db_tag(db) == EXTENDED_TAG) { + DRM_DEBUG_KMS("found extended tag block = %d\n", + db[1]); + switch (db[1]) { + case VIDEO_CAPABILITY_EXTENDED_DATA_BLOCK: + drm_extract_vcdb_info(connector, db); + break; + case HDR_STATIC_METADATA_EXTENDED_DATA_BLOCK: + drm_extract_hdr_db(connector, db); + break; + case COLORIMETRY_EXTENDED_DATA_BLOCK: + drm_extract_clrmetry_db(connector, db); + break; + default: + break; + } + } + } + } +} + +static u8 * +drm_edid_find_extended_tag_block(struct edid *edid, int blk_id) +{ + u8 *db = NULL; + u8 *cea = NULL; + + if (!edid) { + pr_err("%s: invalid input\n", __func__); + return NULL; + } + + cea = drm_find_cea_extension(edid); + + if (cea && cea_revision(cea) >= 3) { + int i, start, end; + + if (cea_db_offsets(cea, &start, &end)) + return NULL; + + for_each_cea_db(cea, i, start, end) { + db = &cea[i]; + if ((cea_db_tag(db) == EXTENDED_TAG) && + (db[1] == blk_id)) + return db; + } + } + return NULL; +} + +/* + * add_YCbCr420VDB_modes - add the modes found in Ycbcr420 VDB block + * @connector: connector corresponding to the HDMI sink + * @edid: handle to the EDID structure + * Parses the YCbCr420 VDB block and adds the modes to @connector. + */ +static int +add_YCbCr420VDB_modes(struct drm_connector *connector, struct edid *edid) +{ + + const u8 *db = NULL; + u32 i = 0; + u32 modes = 0; + u32 video_format = 0; + u8 len = 0; + + /*Find the YCbCr420 VDB*/ + db = drm_edid_find_extended_tag_block(edid, Y420_VIDEO_DATA_BLOCK); + /* Offset to byte 3 */ + if (db) { + len = db[0] & 0x1F; + db += 2; + for (i = 0; i < len - 1; i++) { + struct drm_display_mode *mode; + + video_format = *(db + i) & 0x7F; + mode = drm_display_mode_from_vic_index(connector, + db, len-1, i); + if (mode) { + DRM_DEBUG_KMS("Adding mode for vic = %d\n", + video_format); + drm_mode_probed_add(connector, mode); + modes++; + } + } + } + return modes; +} + +static void monitor_name(struct detailed_timing *t, void *data) { if (t->data.other_data.type == EDID_DETAIL_MONITOR_NAME) @@ -3282,6 +3846,9 @@ /* HDMI Vendor-Specific Data Block */ if (cea_db_is_hdmi_vsdb(db)) parse_hdmi_vsdb(connector, db); + /* HDMI Forum Vendor-Specific Data Block */ + else if (cea_db_is_hdmi_hf_vsdb(db)) + parse_hdmi_hf_vsdb(connector, db); break; default: break; @@ -3738,6 +4305,10 @@ info->color_formats |= DRM_COLOR_FORMAT_YCRCB422; } + /* Extract audio and video latency fields for the sink */ + drm_hdmi_extract_vsdbs_info(connector, edid); + /* Extract info from extended tag blocks */ + drm_hdmi_extract_extended_blk_info(connector, edid); /* HDMI deep color modes supported? Assign to info, if so */ drm_assign_hdmi_deep_color_info(edid, info, connector); @@ -3780,6 +4351,148 @@ info->color_formats |= DRM_COLOR_FORMAT_YCRCB422; } +static int validate_displayid(u8 *displayid, int length, int idx) +{ + int i; + u8 csum = 0; + struct displayid_hdr *base; + + base = (struct displayid_hdr *)&displayid[idx]; + + DRM_DEBUG_KMS("base revision 0x%x, length %d, %d %d\n", + base->rev, base->bytes, base->prod_id, base->ext_count); + + if (base->bytes + 5 > length - idx) + return -EINVAL; + for (i = idx; i <= base->bytes + 5; i++) + csum += displayid[i]; + + if (csum) { + DRM_ERROR("DisplayID checksum invalid, remainder is %d\n", + csum); + return -EINVAL; + } + return 0; +} + +static struct drm_display_mode * +drm_mode_displayid_detailed(struct drm_device *dev, +struct displayid_detailed_timings_1 *timings) +{ + struct drm_display_mode *mode; + unsigned pixel_clock = (timings->pixel_clock[0] | + (timings->pixel_clock[1] << 8) | + (timings->pixel_clock[2] << 16)); + unsigned hactive = (timings->hactive[0] | timings->hactive[1] << 8) + 1; + unsigned hblank = + (timings->hblank[0] | + timings->hblank[1] << 8) + 1; + unsigned hsync = (timings->hsync[0] | + (timings->hsync[1] & 0x7f) << 8) + 1; + unsigned hsync_width = (timings->hsw[0] | timings->hsw[1] << 8) + 1; + unsigned vactive = (timings->vactive[0] | timings->vactive[1] << 8) + 1; + unsigned vblank = + (timings->vblank[0] | + timings->vblank[1] << 8) + 1; + unsigned vsync = + (timings->vsync[0] | + (timings->vsync[1] & 0x7f) << 8) + 1; + unsigned vsync_width = (timings->vsw[0] | timings->vsw[1] << 8) + 1; + bool hsync_positive = (timings->hsync[1] >> 7) & 0x1; + bool vsync_positive = (timings->vsync[1] >> 7) & 0x1; + + mode = drm_mode_create(dev); + if (!mode) + return NULL; + + mode->clock = pixel_clock * 10; + mode->hdisplay = hactive; + mode->hsync_start = mode->hdisplay + hsync; + mode->hsync_end = mode->hsync_start + hsync_width; + mode->htotal = mode->hdisplay + hblank; + + mode->vdisplay = vactive; + mode->vsync_start = mode->vdisplay + vsync; + mode->vsync_end = mode->vsync_start + vsync_width; + mode->vtotal = mode->vdisplay + vblank; + + mode->flags = 0; + mode->flags |= hsync_positive ? + DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC; + mode->flags |= vsync_positive ? + DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC; + mode->type = DRM_MODE_TYPE_DRIVER; + + if (timings->flags & 0x80) + mode->type |= DRM_MODE_TYPE_PREFERRED; + mode->vrefresh = drm_mode_vrefresh(mode); + drm_mode_set_name(mode); + + return mode; +} + +static int add_displayid_detailed_1_modes(struct drm_connector *connector, + struct displayid_block *block) +{ + struct displayid_detailed_timing_block *det = + (struct displayid_detailed_timing_block *)block; + int i; + int num_timings; + struct drm_display_mode *newmode; + int num_modes = 0; + /* blocks must be multiple of 20 bytes length */ + if (block->num_bytes % 20) + return 0; + + num_timings = block->num_bytes / 20; + for (i = 0; i < num_timings; i++) { + struct displayid_detailed_timings_1 *timings = &det->timings[i]; + + newmode = drm_mode_displayid_detailed(connector->dev, timings); + if (!newmode) + continue; + + drm_mode_probed_add(connector, newmode); + num_modes++; + } + return num_modes; +} + +static int add_displayid_detailed_modes(struct drm_connector *connector, + struct edid *edid) +{ + u8 *displayid; + int ret; + int idx = 1; + int length = EDID_LENGTH; + struct displayid_block *block; + int num_modes = 0; + + displayid = drm_find_displayid_extension(edid); + if (!displayid) + return 0; + + ret = validate_displayid(displayid, length, idx); + if (ret) + return 0; + + idx += sizeof(struct displayid_hdr); + while (block = (struct displayid_block *)&displayid[idx], + idx + sizeof(struct displayid_block) <= length && + idx + sizeof(struct displayid_block) + + block->num_bytes <= length && + block->num_bytes > 0) { + idx += block->num_bytes + sizeof(struct displayid_block); + switch (block->tag) { + case DATA_BLOCK_TYPE_1_DETAILED_TIMING: + num_modes += add_displayid_detailed_1_modes(connector, + block); + break; + } + } + return num_modes; +} + /** * drm_add_edid_modes - add modes from EDID data, if available * @connector: connector we're probing @@ -3825,6 +4538,8 @@ num_modes += add_established_modes(connector, edid); num_modes += add_cea_modes(connector, edid); num_modes += add_alternate_cea_modes(connector, edid); + num_modes += add_displayid_detailed_modes(connector, edid); + num_modes += add_YCbCr420VDB_modes(connector, edid); if (edid->features & DRM_EDID_FEATURE_DEFAULT_GTF) num_modes += add_inferred_modes(connector, edid); @@ -4037,44 +4752,11 @@ } EXPORT_SYMBOL(drm_hdmi_vendor_infoframe_from_display_mode); -static int drm_parse_display_id(struct drm_connector *connector, - u8 *displayid, int length, - bool is_edid_extension) +static int drm_parse_tiled_block(struct drm_connector *connector, + struct displayid_block *block) { - /* if this is an EDID extension the first byte will be 0x70 */ - int idx = 0; - struct displayid_hdr *base; - struct displayid_block *block; - u8 csum = 0; - int i; - - if (is_edid_extension) - idx = 1; - - base = (struct displayid_hdr *)&displayid[idx]; - - DRM_DEBUG_KMS("base revision 0x%x, length %d, %d %d\n", - base->rev, base->bytes, base->prod_id, base->ext_count); - - if (base->bytes + 5 > length - idx) - return -EINVAL; - - for (i = idx; i <= base->bytes + 5; i++) { - csum += displayid[i]; - } - if (csum) { - DRM_ERROR("DisplayID checksum invalid, remainder is %d\n", csum); - return -EINVAL; - } - - block = (struct displayid_block *)&displayid[idx + 4]; - DRM_DEBUG_KMS("block id %d, rev %d, len %d\n", - block->tag, block->rev, block->num_bytes); - - switch (block->tag) { - case DATA_BLOCK_TILED_DISPLAY: { - struct displayid_tiled_block *tile = (struct displayid_tiled_block *)block; - + struct displayid_tiled_block *tile = + (struct displayid_tiled_block *)block; u16 w, h; u8 tile_v_loc, tile_h_loc; u8 num_v_tile, num_h_tile; @@ -4103,31 +4785,73 @@ DRM_DEBUG_KMS("tile_size %d x %d\n", w + 1, h + 1); DRM_DEBUG_KMS("topo num tiles %dx%d, location %dx%d\n", num_h_tile + 1, num_v_tile + 1, tile_h_loc, tile_v_loc); - DRM_DEBUG_KMS("vend %c%c%c\n", tile->topology_id[0], tile->topology_id[1], tile->topology_id[2]); + DRM_DEBUG_KMS("vend %c%c%c\n", tile->topology_id[0], + tile->topology_id[1], tile->topology_id[2]); tg = drm_mode_get_tile_group(connector->dev, tile->topology_id); - if (!tg) { - tg = drm_mode_create_tile_group(connector->dev, tile->topology_id); - } + if (!tg) + tg = drm_mode_create_tile_group(connector->dev, + tile->topology_id); + if (!tg) return -ENOMEM; if (connector->tile_group != tg) { /* if we haven't got a pointer, - take the reference, drop ref to old tile group */ - if (connector->tile_group) { - drm_mode_put_tile_group(connector->dev, connector->tile_group); - } + * take the reference, drop ref to old tile group + */ + if (connector->tile_group) + drm_mode_put_tile_group(connector->dev, + connector->tile_group); + connector->tile_group = tg; } else /* if same tile group, then release the ref we just took. */ drm_mode_put_tile_group(connector->dev, tg); + return 0; } + +static int drm_parse_display_id(struct drm_connector *connector, + u8 *displayid, int length, + bool is_edid_extension) +{ + /* if this is an EDID extension the first byte will be 0x70 */ + int idx = 0; + struct displayid_block *block; + int ret; + + if (is_edid_extension) + idx = 1; + + ret = validate_displayid(displayid, length, idx); + if (ret) + return ret; + + idx += sizeof(struct displayid_hdr); + while (block = (struct displayid_block *)&displayid[idx], + idx + sizeof(struct displayid_block) <= length && + idx + sizeof(struct displayid_block) + + block->num_bytes <= length && + block->num_bytes > 0) { + idx += block->num_bytes + sizeof(struct displayid_block); + DRM_DEBUG_KMS("block id 0x%x, rev %d, len %d\n", + block->tag, block->rev, block->num_bytes); + + switch (block->tag) { + case DATA_BLOCK_TILED_DISPLAY: + ret = drm_parse_tiled_block(connector, block); + if (ret) + return ret; + break; + case DATA_BLOCK_TYPE_1_DETAILED_TIMING: + /* handled in mode gathering code. */ break; default: - printk("unknown displayid tag %d\n", block->tag); + DRM_DEBUG_KMS("found DisplayID tag 0x%x, unhandled\n", + block->tag); break; } + } return 0; } diff -ruw linux-4.4.115/drivers/gpu/drm/drm_ioctl.c linux-4.4.115-fbx/drivers/gpu/drm/drm_ioctl.c --- linux-4.4.115/drivers/gpu/drm/drm_ioctl.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/drm_ioctl.c 2019-10-29 09:26:23.605202806 +0100 @@ -312,6 +312,9 @@ case DRM_CAP_ADDFB2_MODIFIERS: req->value = dev->mode_config.allow_fb_modifiers; break; + case DRM_CAP_CRTC_IN_VBLANK_EVENT: + req->value = 1; + break; default: return -EINVAL; } diff -ruw linux-4.4.115/drivers/gpu/drm/drm_irq.c linux-4.4.115-fbx/drivers/gpu/drm/drm_irq.c --- linux-4.4.115/drivers/gpu/drm/drm_irq.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/drm_irq.c 2019-10-29 09:26:23.605202806 +0100 @@ -1075,6 +1075,7 @@ e->pipe = pipe; e->event.sequence = drm_vblank_count(dev, pipe); + e->event.crtc_id = 0; list_add_tail(&e->base.link, &dev->vblank_event_list); } EXPORT_SYMBOL(drm_arm_vblank_event); @@ -1098,7 +1099,15 @@ void drm_crtc_arm_vblank_event(struct drm_crtc *crtc, struct drm_pending_vblank_event *e) { - drm_arm_vblank_event(crtc->dev, drm_crtc_index(crtc), e); + struct drm_device *dev = crtc->dev; + unsigned int pipe = drm_crtc_index(crtc); + + assert_spin_locked(&dev->event_lock); + + e->pipe = pipe; + e->event.sequence = drm_vblank_count(dev, pipe); + e->event.crtc_id = crtc->base.id; + list_add_tail(&e->base.link, &dev->vblank_event_list); } EXPORT_SYMBOL(drm_crtc_arm_vblank_event); @@ -1127,6 +1136,7 @@ now = get_drm_timestamp(); } e->pipe = pipe; + e->event.crtc_id = 0; send_vblank_event(dev, e, seq, &now); } EXPORT_SYMBOL(drm_send_vblank_event); @@ -1144,7 +1154,20 @@ void drm_crtc_send_vblank_event(struct drm_crtc *crtc, struct drm_pending_vblank_event *e) { - drm_send_vblank_event(crtc->dev, drm_crtc_index(crtc), e); + struct drm_device *dev = crtc->dev; + unsigned int seq, pipe = drm_crtc_index(crtc); + struct timeval now; + + if (dev->num_crtcs > 0) { + seq = drm_vblank_count_and_time(dev, pipe, &now); + } else { + seq = 0; + + now = get_drm_timestamp(); + } + e->pipe = pipe; + e->event.crtc_id = crtc->base.id; + send_vblank_event(dev, e, seq, &now); } EXPORT_SYMBOL(drm_crtc_send_vblank_event); diff -ruw linux-4.4.115/drivers/gpu/drm/drm_mipi_dsi.c linux-4.4.115-fbx/drivers/gpu/drm/drm_mipi_dsi.c --- linux-4.4.115/drivers/gpu/drm/drm_mipi_dsi.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/drm_mipi_dsi.c 2019-01-22 16:16:23.375245247 +0100 @@ -335,7 +335,7 @@ return -EINVAL; memset(packet, 0, sizeof(*packet)); - packet->header[0] = ((msg->channel & 0x3) << 6) | (msg->type & 0x3f); + packet->header[2] = ((msg->channel & 0x3) << 6) | (msg->type & 0x3f); /* TODO: compute ECC if hardware support is not available */ @@ -347,16 +347,16 @@ * and 2. */ if (mipi_dsi_packet_format_is_long(msg->type)) { - packet->header[1] = (msg->tx_len >> 0) & 0xff; - packet->header[2] = (msg->tx_len >> 8) & 0xff; + packet->header[0] = (msg->tx_len >> 0) & 0xff; + packet->header[1] = (msg->tx_len >> 8) & 0xff; packet->payload_length = msg->tx_len; packet->payload = msg->tx_buf; } else { const u8 *tx = msg->tx_buf; - packet->header[1] = (msg->tx_len > 0) ? tx[0] : 0; - packet->header[2] = (msg->tx_len > 1) ? tx[1] : 0; + packet->header[0] = (msg->tx_len > 0) ? tx[0] : 0; + packet->header[1] = (msg->tx_len > 1) ? tx[1] : 0; } packet->size = sizeof(packet->header) + packet->payload_length; diff -ruw linux-4.4.115/drivers/gpu/drm/drm_mm.c linux-4.4.115-fbx/drivers/gpu/drm/drm_mm.c --- linux-4.4.115/drivers/gpu/drm/drm_mm.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/drm_mm.c 2019-01-22 16:16:23.375245247 +0100 @@ -46,6 +46,8 @@ #include #include #include +#include +#include /** * DOC: Overview @@ -73,7 +75,8 @@ * allocations and avoiding too much fragmentation. This means free space * searches are O(num_holes). Given that all the fancy features drm_mm supports * something better would be fairly complex and since gfx thrashing is a fairly - * steep cliff not a real concern. Removing a node again is O(1). + * steep cliff not a real concern. Removing a node again is O(1). With the + * rbtree to track free holes, free hole search becomes O(log(num_holes)). * * drm_mm supports a few features: Alignment and range restrictions can be * supplied. Further more every &drm_mm_node has a color value (which is just an @@ -103,6 +106,98 @@ u64 end, enum drm_mm_search_flags flags); +#define START(node) ((node)->start) +#define LAST(node) ((node)->start + (node)->size - 1) + +INTERVAL_TREE_DEFINE(struct drm_mm_node, rb, + u64, __subtree_last, + START, LAST, static inline, drm_mm_interval_tree) + +struct drm_mm_node * +drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last) +{ + return drm_mm_interval_tree_iter_first(&mm->interval_tree, + start, last); +} +EXPORT_SYMBOL(drm_mm_interval_first); + +struct drm_mm_node * +drm_mm_interval_next(struct drm_mm_node *node, u64 start, u64 last) +{ + return drm_mm_interval_tree_iter_next(node, start, last); +} +EXPORT_SYMBOL(drm_mm_interval_next); + +static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node, + struct drm_mm_node *node) +{ + struct drm_mm *mm = hole_node->mm; + struct rb_node **link, *rb; + struct drm_mm_node *parent; + + node->__subtree_last = LAST(node); + + if (hole_node->allocated) { + rb = &hole_node->rb; + while (rb) { + parent = rb_entry(rb, struct drm_mm_node, rb); + if (parent->__subtree_last >= node->__subtree_last) + break; + + parent->__subtree_last = node->__subtree_last; + rb = rb_parent(rb); + } + + rb = &hole_node->rb; + link = &hole_node->rb.rb_right; + } else { + rb = NULL; + link = &mm->interval_tree.rb_node; + } + + while (*link) { + rb = *link; + parent = rb_entry(rb, struct drm_mm_node, rb); + if (parent->__subtree_last < node->__subtree_last) + parent->__subtree_last = node->__subtree_last; + if (node->start < parent->start) + link = &parent->rb.rb_left; + else + link = &parent->rb.rb_right; + } + + rb_link_node(&node->rb, rb, link); + rb_insert_augmented(&node->rb, + &mm->interval_tree, + &drm_mm_interval_tree_augment); +} + +static void +rb_insert_hole_node(struct drm_mm_node *hole_node, struct drm_mm *mm) +{ + struct rb_node **new = &(mm->holes_tree.rb_node); + struct rb_node *parent = NULL; + struct drm_mm_node *cur; + + while (*new) { + parent = *new; + cur = rb_entry(parent, struct drm_mm_node, hole_node); + + if (__drm_mm_hole_node_start(hole_node) + < __drm_mm_hole_node_start(cur)) + new = &parent->rb_left; + else + new = &parent->rb_right; + } + rb_link_node(&hole_node->hole_node, parent, new); + rb_insert_color(&hole_node->hole_node, &mm->holes_tree); +} + +static void rb_erase_hole_node(struct drm_mm_node *hole_node, struct drm_mm *mm) +{ + rb_erase(&hole_node->hole_node, &mm->holes_tree); +} + static void drm_mm_insert_helper(struct drm_mm_node *hole_node, struct drm_mm_node *node, u64 size, unsigned alignment, @@ -142,6 +237,7 @@ if (adj_start == hole_start) { hole_node->hole_follows = 0; list_del(&hole_node->hole_stack); + rb_erase_hole_node(hole_node, mm); } node->start = adj_start; @@ -150,14 +246,16 @@ node->color = color; node->allocated = 1; - INIT_LIST_HEAD(&node->hole_stack); list_add(&node->node_list, &hole_node->node_list); + drm_mm_interval_tree_add_node(hole_node, node); + BUG_ON(node->start + node->size > adj_end); node->hole_follows = 0; if (__drm_mm_hole_node_start(node) < hole_end) { list_add(&node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(node, mm); node->hole_follows = 1; } } @@ -178,40 +276,55 @@ */ int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node) { - struct drm_mm_node *hole; u64 end = node->start + node->size; - u64 hole_start; - u64 hole_end; + struct drm_mm_node *hole; + u64 hole_start, hole_end; - BUG_ON(node == NULL); + if (WARN_ON(node->size == 0)) + return -EINVAL; /* Find the relevant hole to add our node to */ - drm_mm_for_each_hole(hole, mm, hole_start, hole_end) { + hole = drm_mm_interval_tree_iter_first(&mm->interval_tree, + node->start, ~(u64)0); + if (hole) { + if (hole->start < end) + return -ENOSPC; + } else { + hole = list_entry(&mm->head_node.node_list, + typeof(*hole), node_list); + } + + hole = list_last_entry(&hole->node_list, typeof(*hole), node_list); + if (!hole->hole_follows) + return -ENOSPC; + + hole_start = __drm_mm_hole_node_start(hole); + hole_end = __drm_mm_hole_node_end(hole); if (hole_start > node->start || hole_end < end) - continue; + return -ENOSPC; node->mm = mm; node->allocated = 1; - INIT_LIST_HEAD(&node->hole_stack); list_add(&node->node_list, &hole->node_list); + drm_mm_interval_tree_add_node(hole, node); + if (node->start == hole_start) { hole->hole_follows = 0; - list_del_init(&hole->hole_stack); + list_del(&hole->hole_stack); + rb_erase_hole_node(hole, mm); } node->hole_follows = 0; if (end != hole_end) { list_add(&node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(node, mm); node->hole_follows = 1; } return 0; } - - return -ENOSPC; -} EXPORT_SYMBOL(drm_mm_reserve_node); /** @@ -237,6 +350,9 @@ { struct drm_mm_node *hole_node; + if (WARN_ON(size == 0)) + return -EINVAL; + hole_node = drm_mm_search_free_generic(mm, size, alignment, color, sflags); if (!hole_node) @@ -287,6 +403,7 @@ if (adj_start == hole_start) { hole_node->hole_follows = 0; list_del(&hole_node->hole_stack); + rb_erase_hole_node(hole_node, mm); } node->start = adj_start; @@ -295,9 +412,10 @@ node->color = color; node->allocated = 1; - INIT_LIST_HEAD(&node->hole_stack); list_add(&node->node_list, &hole_node->node_list); + drm_mm_interval_tree_add_node(hole_node, node); + BUG_ON(node->start < start); BUG_ON(node->start < adj_start); BUG_ON(node->start + node->size > adj_end); @@ -306,6 +424,7 @@ node->hole_follows = 0; if (__drm_mm_hole_node_start(node) < hole_end) { list_add(&node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(node, mm); node->hole_follows = 1; } } @@ -336,6 +455,9 @@ { struct drm_mm_node *hole_node; + if (WARN_ON(size == 0)) + return -EINVAL; + hole_node = drm_mm_search_free_in_range_generic(mm, size, alignment, color, start, end, sflags); @@ -375,6 +497,7 @@ BUG_ON(__drm_mm_hole_node_start(node) == __drm_mm_hole_node_end(node)); list_del(&node->hole_stack); + rb_erase_hole_node(node, mm); } else BUG_ON(__drm_mm_hole_node_start(node) != __drm_mm_hole_node_end(node)); @@ -383,9 +506,11 @@ if (!prev_node->hole_follows) { prev_node->hole_follows = 1; list_add(&prev_node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(prev_node, mm); } else list_move(&prev_node->hole_stack, &mm->hole_stack); + drm_mm_interval_tree_remove(node, &mm->interval_tree); list_del(&node->node_list); node->allocated = 0; } @@ -408,6 +533,53 @@ return end >= start + size; } +static struct drm_mm_node *get_first_hole(const struct drm_mm *mm, + enum drm_mm_search_flags flags) +{ + if (flags & DRM_MM_SEARCH_BOTTOM_UP) { + struct rb_node *node = rb_first(&mm->holes_tree); + + if (!node) + return NULL; + + return rb_entry(node, struct drm_mm_node, hole_node); + } else if (flags & DRM_MM_SEARCH_BELOW) { + return list_entry((mm)->hole_stack.prev, + struct drm_mm_node, hole_stack); + } else { + return list_entry((mm)->hole_stack.next, + struct drm_mm_node, hole_stack); + } +} + +static struct drm_mm_node *get_next_hole(struct drm_mm_node *entry, + enum drm_mm_search_flags flags) +{ + if (flags & DRM_MM_SEARCH_BOTTOM_UP) { + struct rb_node *node = rb_next(&entry->hole_node); + + if (!node) + return NULL; + + return rb_entry(node, struct drm_mm_node, hole_node); + } else if (flags & DRM_MM_SEARCH_BELOW) { + return list_entry(entry->hole_stack.prev, + struct drm_mm_node, hole_stack); + } else { + return list_entry(entry->hole_stack.next, + struct drm_mm_node, hole_stack); + } +} + +static bool drm_mm_hole_traversal_condition(const struct drm_mm *mm, + struct drm_mm_node *entry, enum drm_mm_search_flags flags) +{ + if (flags & DRM_MM_SEARCH_BOTTOM_UP) + return entry ? 1 : 0; + else + return (&entry->hole_stack != &(mm)->hole_stack) ? 1 : 0; +} + static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, u64 size, unsigned alignment, @@ -425,9 +597,14 @@ best = NULL; best_size = ~0UL; - __drm_mm_for_each_hole(entry, mm, adj_start, adj_end, - flags & DRM_MM_SEARCH_BELOW) { - u64 hole_size = adj_end - adj_start; + for (entry = get_first_hole(mm, flags); + drm_mm_hole_traversal_condition(mm, entry, flags); + entry = get_next_hole(entry, flags)) { + u64 hole_size; + + adj_start = drm_mm_hole_node_start(entry); + adj_end = drm_mm_hole_node_end(entry); + hole_size = adj_end - adj_start; if (mm->color_adjust) { mm->color_adjust(entry, color, &adj_start, &adj_end); @@ -469,9 +646,14 @@ best = NULL; best_size = ~0UL; - __drm_mm_for_each_hole(entry, mm, adj_start, adj_end, - flags & DRM_MM_SEARCH_BELOW) { - u64 hole_size = adj_end - adj_start; + for (entry = get_first_hole(mm, flags); + drm_mm_hole_traversal_condition(mm, entry, flags); + entry = get_next_hole(entry, flags)) { + u64 hole_size; + + adj_start = drm_mm_hole_node_start(entry); + adj_end = drm_mm_hole_node_end(entry); + hole_size = adj_end - adj_start; if (mm->color_adjust) { mm->color_adjust(entry, color, &adj_start, &adj_end); @@ -510,14 +692,21 @@ { list_replace(&old->node_list, &new->node_list); list_replace(&old->hole_stack, &new->hole_stack); + rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree); new->hole_follows = old->hole_follows; new->mm = old->mm; new->start = old->start; new->size = old->size; new->color = old->color; + new->__subtree_last = old->__subtree_last; old->allocated = 0; new->allocated = 1; + + if (old->hole_follows) + rb_replace_node(&old->hole_node, &new->hole_node, + &old->mm->holes_tree); + } EXPORT_SYMBOL(drm_mm_replace_node); @@ -742,7 +931,6 @@ /* Clever trick to avoid a special case in the free hole tracking. */ INIT_LIST_HEAD(&mm->head_node.node_list); - INIT_LIST_HEAD(&mm->head_node.hole_stack); mm->head_node.hole_follows = 1; mm->head_node.scanned_block = 0; mm->head_node.scanned_prev_free = 0; @@ -752,7 +940,10 @@ mm->head_node.size = start - mm->head_node.start; list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack); + mm->interval_tree = RB_ROOT; mm->color_adjust = NULL; + mm->holes_tree = RB_ROOT; + rb_insert_hole_node(&mm->head_node, mm); } EXPORT_SYMBOL(drm_mm_init); diff -ruw linux-4.4.115/drivers/gpu/drm/drm_probe_helper.c linux-4.4.115-fbx/drivers/gpu/drm/drm_probe_helper.c --- linux-4.4.115/drivers/gpu/drm/drm_probe_helper.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/drm_probe_helper.c 2019-10-29 09:26:23.605202806 +0100 @@ -199,20 +199,7 @@ goto prune; } -#ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE - count = drm_load_edid_firmware(connector); - if (count == 0) -#endif - { - if (connector->override_edid) { - struct edid *edid = (struct edid *) connector->edid_blob_ptr->data; - - count = drm_add_edid_modes(connector, edid); - drm_edid_to_eld(connector, edid); - } else count = (*connector_funcs->get_modes)(connector); - } - if (count == 0 && connector->status == connector_status_connected) count = drm_add_modes_noedid(connector, 1024, 768); count += drm_helper_probe_add_cmdline_mode(connector); diff -ruw linux-4.4.115/drivers/gpu/drm/Kconfig linux-4.4.115-fbx/drivers/gpu/drm/Kconfig --- linux-4.4.115/drivers/gpu/drm/Kconfig 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/Kconfig 2019-07-19 15:49:45.457091212 +0200 @@ -8,6 +8,7 @@ tristate "Direct Rendering Manager (XFree86 4.1.0 and higher DRI support)" depends on (AGP || AGP=n) && !EMULATED_CMPXCHG && MMU && HAS_DMA select HDMI + select FB select FB_CMDLINE select I2C select I2C_ALGOBIT @@ -52,7 +53,7 @@ depends on DRM select DRM_KMS_HELPER select DRM_KMS_FB_HELPER - default y + default n help Choose this option if you have a need for the legacy fbdev support. Note that this support also provides the linux console @@ -62,7 +63,7 @@ config DRM_LOAD_EDID_FIRMWARE bool "Allow to specify an EDID data set instead of probing for it" - depends on DRM_KMS_HELPER + depends on DRM help Say Y here, if you want to use EDID data to be loaded from the /lib/firmware directory or one of the provided built-in @@ -266,3 +267,5 @@ source "drivers/gpu/drm/imx/Kconfig" source "drivers/gpu/drm/vc4/Kconfig" + +source "drivers/gpu/drm/msm-hyp/Kconfig" diff -ruw linux-4.4.115/drivers/gpu/drm/Makefile linux-4.4.115-fbx/drivers/gpu/drm/Makefile --- linux-4.4.115/drivers/gpu/drm/Makefile 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/Makefile 2019-07-19 15:49:45.461091250 +0200 @@ -20,12 +20,12 @@ drm-$(CONFIG_DRM_PANEL) += drm_panel.o drm-$(CONFIG_OF) += drm_of.o drm-$(CONFIG_AGP) += drm_agpsupport.o +drm-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o drm-y += $(drm-m) drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \ drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o -drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o drm_kms_helper-$(CONFIG_DRM_KMS_CMA_HELPER) += drm_fb_cma_helper.o @@ -68,6 +68,7 @@ obj-$(CONFIG_DRM_BOCHS) += bochs/ obj-$(CONFIG_DRM_VIRTIO_GPU) += virtio/ obj-$(CONFIG_DRM_MSM) += msm/ +obj-$(CONFIG_DRM_MSM_HYP) += msm-hyp/ obj-$(CONFIG_DRM_TEGRA) += tegra/ obj-$(CONFIG_DRM_STI) += sti/ obj-$(CONFIG_DRM_IMX) += imx/ diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/a3xx_gpu.c linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a3xx_gpu.c --- linux-4.4.115/drivers/gpu/drm/msm/adreno/a3xx_gpu.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a3xx_gpu.c 2019-01-22 16:16:23.479246189 +0100 @@ -2,7 +2,7 @@ * Copyright (C) 2013 Red Hat * Author: Rob Clark * - * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (c) 2014,2018 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published by @@ -40,10 +40,11 @@ extern bool hang_debug; static void a3xx_dump(struct msm_gpu *gpu); +static bool a3xx_idle(struct msm_gpu *gpu); -static void a3xx_me_init(struct msm_gpu *gpu) +static bool a3xx_me_init(struct msm_gpu *gpu) { - struct msm_ringbuffer *ring = gpu->rb; + struct msm_ringbuffer *ring = gpu->rb[0]; OUT_PKT3(ring, CP_ME_INIT, 17); OUT_RING(ring, 0x000003f7); @@ -64,8 +65,8 @@ OUT_RING(ring, 0x00000000); OUT_RING(ring, 0x00000000); - gpu->funcs->flush(gpu); - gpu->funcs->idle(gpu); + gpu->funcs->flush(gpu, ring); + return a3xx_idle(gpu); } static int a3xx_hw_init(struct msm_gpu *gpu) @@ -294,9 +295,7 @@ /* clear ME_HALT to start micro engine */ gpu_write(gpu, REG_AXXX_CP_ME_CNTL, 0); - a3xx_me_init(gpu); - - return 0; + return a3xx_me_init(gpu) ? 0 : -EINVAL; } static void a3xx_recover(struct msm_gpu *gpu) @@ -330,17 +329,22 @@ kfree(a3xx_gpu); } -static void a3xx_idle(struct msm_gpu *gpu) +static bool a3xx_idle(struct msm_gpu *gpu) { /* wait for ringbuffer to drain: */ - adreno_idle(gpu); + if (!adreno_idle(gpu, gpu->rb[0])) + return false; /* then wait for GPU to finish: */ if (spin_until(!(gpu_read(gpu, REG_A3XX_RBBM_STATUS) & - A3XX_RBBM_STATUS_GPU_BUSY))) + A3XX_RBBM_STATUS_GPU_BUSY))) { DRM_ERROR("%s: timeout waiting for GPU to idle!\n", gpu->name); /* TODO maybe we need to reset GPU here to recover from hang? */ + return false; + } + + return true; } static irqreturn_t a3xx_irq(struct msm_gpu *gpu) @@ -402,10 +406,8 @@ #ifdef CONFIG_DEBUG_FS static void a3xx_show(struct msm_gpu *gpu, struct seq_file *m) { - gpu->funcs->pm_resume(gpu); seq_printf(m, "status: %08x\n", gpu_read(gpu, REG_A3XX_RBBM_STATUS)); - gpu->funcs->pm_suspend(gpu); adreno_show(gpu, m); } #endif @@ -419,91 +421,13 @@ } /* Register offset defines for A3XX */ static const unsigned int a3xx_register_offsets[REG_ADRENO_REGISTER_MAX] = { - REG_ADRENO_DEFINE(REG_ADRENO_CP_DEBUG, REG_AXXX_CP_DEBUG), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_WADDR, REG_AXXX_CP_ME_RAM_WADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_DATA, REG_AXXX_CP_ME_RAM_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_DATA, - REG_A3XX_CP_PFP_UCODE_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_ADDR, - REG_A3XX_CP_PFP_UCODE_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_WFI_PEND_CTR, REG_A3XX_CP_WFI_PEND_CTR), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_AXXX_CP_RB_BASE), + REG_ADRENO_SKIP(REG_ADRENO_CP_RB_BASE_HI), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_AXXX_CP_RB_RPTR_ADDR), + REG_ADRENO_SKIP(REG_ADRENO_CP_RB_RPTR_ADDR_HI), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_AXXX_CP_RB_RPTR), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_AXXX_CP_RB_WPTR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_CTRL, REG_A3XX_CP_PROTECT_CTRL), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_CNTL, REG_AXXX_CP_ME_CNTL), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_AXXX_CP_RB_CNTL), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BASE, REG_AXXX_CP_IB1_BASE), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BUFSZ, REG_AXXX_CP_IB1_BUFSZ), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BASE, REG_AXXX_CP_IB2_BASE), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BUFSZ, REG_AXXX_CP_IB2_BUFSZ), - REG_ADRENO_DEFINE(REG_ADRENO_CP_TIMESTAMP, REG_AXXX_CP_SCRATCH_REG0), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_RADDR, REG_AXXX_CP_ME_RAM_RADDR), - REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_ADDR, REG_AXXX_SCRATCH_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_UMSK, REG_AXXX_SCRATCH_UMSK), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_ADDR, REG_A3XX_CP_ROQ_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_DATA, REG_A3XX_CP_ROQ_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_ADDR, REG_A3XX_CP_MERCIU_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA, REG_A3XX_CP_MERCIU_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA2, REG_A3XX_CP_MERCIU_DATA2), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_ADDR, REG_A3XX_CP_MEQ_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_DATA, REG_A3XX_CP_MEQ_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_HW_FAULT, REG_A3XX_CP_HW_FAULT), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_STATUS, - REG_A3XX_CP_PROTECT_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_STATUS, REG_A3XX_RBBM_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_CTL, - REG_A3XX_RBBM_PERFCTR_CTL), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD0, - REG_A3XX_RBBM_PERFCTR_LOAD_CMD0), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD1, - REG_A3XX_RBBM_PERFCTR_LOAD_CMD1), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_PWR_1_LO, - REG_A3XX_RBBM_PERFCTR_PWR_1_LO), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_MASK, REG_A3XX_RBBM_INT_0_MASK), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_STATUS, - REG_A3XX_RBBM_INT_0_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_ERROR_STATUS, - REG_A3XX_RBBM_AHB_ERROR_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_CMD, REG_A3XX_RBBM_AHB_CMD), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_CLEAR_CMD, - REG_A3XX_RBBM_INT_CLEAR_CMD), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_CLOCK_CTL, REG_A3XX_RBBM_CLOCK_CTL), - REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_SEL, - REG_A3XX_VPC_VPC_DEBUG_RAM_SEL), - REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_READ, - REG_A3XX_VPC_VPC_DEBUG_RAM_READ), - REG_ADRENO_DEFINE(REG_ADRENO_VSC_SIZE_ADDRESS, - REG_A3XX_VSC_SIZE_ADDRESS), - REG_ADRENO_DEFINE(REG_ADRENO_VFD_CONTROL_0, REG_A3XX_VFD_CONTROL_0), - REG_ADRENO_DEFINE(REG_ADRENO_VFD_INDEX_MAX, REG_A3XX_VFD_INDEX_MAX), - REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_PVT_MEM_ADDR_REG, - REG_A3XX_SP_VS_PVT_MEM_ADDR_REG), - REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_PVT_MEM_ADDR_REG, - REG_A3XX_SP_FS_PVT_MEM_ADDR_REG), - REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_OBJ_START_REG, - REG_A3XX_SP_VS_OBJ_START_REG), - REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_OBJ_START_REG, - REG_A3XX_SP_FS_OBJ_START_REG), - REG_ADRENO_DEFINE(REG_ADRENO_PA_SC_AA_CONFIG, REG_A3XX_PA_SC_AA_CONFIG), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PM_OVERRIDE2, - REG_A3XX_RBBM_PM_OVERRIDE2), - REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_REG2, REG_AXXX_CP_SCRATCH_REG2), - REG_ADRENO_DEFINE(REG_ADRENO_SQ_GPR_MANAGEMENT, - REG_A3XX_SQ_GPR_MANAGEMENT), - REG_ADRENO_DEFINE(REG_ADRENO_SQ_INST_STORE_MANAGMENT, - REG_A3XX_SQ_INST_STORE_MANAGMENT), - REG_ADRENO_DEFINE(REG_ADRENO_TP0_CHICKEN, REG_A3XX_TP0_CHICKEN), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_RBBM_CTL, REG_A3XX_RBBM_RBBM_CTL), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_SW_RESET_CMD, - REG_A3XX_RBBM_SW_RESET_CMD), - REG_ADRENO_DEFINE(REG_ADRENO_UCHE_INVALIDATE0, - REG_A3XX_UCHE_CACHE_INVALIDATE0_REG), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_LO, - REG_A3XX_RBBM_PERFCTR_LOAD_VALUE_LO), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_HI, - REG_A3XX_RBBM_PERFCTR_LOAD_VALUE_HI), }; static const struct adreno_gpu_funcs funcs = { @@ -513,10 +437,10 @@ .pm_suspend = msm_gpu_pm_suspend, .pm_resume = msm_gpu_pm_resume, .recover = a3xx_recover, - .last_fence = adreno_last_fence, + .submitted_fence = adreno_submitted_fence, .submit = adreno_submit, .flush = adreno_flush, - .idle = a3xx_idle, + .active_ring = adreno_active_ring, .irq = a3xx_irq, .destroy = a3xx_destroy, #ifdef CONFIG_DEBUG_FS @@ -539,6 +463,7 @@ struct msm_gpu *gpu; struct msm_drm_private *priv = dev->dev_private; struct platform_device *pdev = priv->gpu_pdev; + struct msm_gpu_config a3xx_config = { 0 }; int ret; if (!pdev) { @@ -556,15 +481,19 @@ adreno_gpu = &a3xx_gpu->base; gpu = &adreno_gpu->base; - a3xx_gpu->pdev = pdev; - gpu->perfcntrs = perfcntrs; gpu->num_perfcntrs = ARRAY_SIZE(perfcntrs); adreno_gpu->registers = a3xx_registers; adreno_gpu->reg_offsets = a3xx_register_offsets; - ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs); + a3xx_config.ioname = MSM_GPU_DEFAULT_IONAME; + a3xx_config.irqname = MSM_GPU_DEFAULT_IRQNAME; + a3xx_config.nr_rings = 1; + a3xx_config.va_start = 0x300000; + a3xx_config.va_end = 0xffffffff; + + ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, &a3xx_config); if (ret) goto fail; @@ -583,7 +512,7 @@ #endif } - if (!gpu->mmu) { + if (!gpu->aspace) { /* TODO we think it is possible to configure the GPU to * restrict access to VRAM carveout. But the required * registers are unknown. For now just bail out and diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/a3xx_gpu.h linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a3xx_gpu.h --- linux-4.4.115/drivers/gpu/drm/msm/adreno/a3xx_gpu.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a3xx_gpu.h 2019-01-22 16:16:23.479246189 +0100 @@ -2,6 +2,8 @@ * Copyright (C) 2013 Red Hat * Author: Rob Clark * + * Copyright (c) 2018 The Linux Foundation. All rights reserved. + * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published by * the Free Software Foundation. @@ -28,7 +30,6 @@ struct a3xx_gpu { struct adreno_gpu base; - struct platform_device *pdev; /* if OCMEM is used for GMEM: */ uint32_t ocmem_base; diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/a3xx.xml.h linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a3xx.xml.h --- linux-4.4.115/drivers/gpu/drm/msm/adreno/a3xx.xml.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a3xx.xml.h 2019-01-22 16:16:23.479246189 +0100 @@ -8,17 +8,19 @@ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00) +- ./adreno.xml ( 431 bytes, from 2016-10-24 21:12:27) +- ./freedreno_copyright.xml ( 1572 bytes, from 2016-10-24 21:12:27) +- ./adreno/a2xx.xml ( 32901 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_common.xml ( 12025 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_pm4.xml ( 19684 bytes, from 2016-10-24 21:12:27) +- ./adreno/a3xx.xml ( 83840 bytes, from 2016-10-24 21:12:27) +- ./adreno/a4xx.xml ( 110708 bytes, from 2016-10-24 21:12:27) +- ./adreno/a5xx.xml ( 81207 bytes, from 2016-10-26 19:36:59) +- ./adreno/ocmem.xml ( 1773 bytes, from 2016-10-24 21:12:27) -Copyright (C) 2013-2015 by the following authors: +Copyright (C) 2013-2016 by the following authors: - Rob Clark (robclark) +- Ilia Mirkin (imirkin) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -111,10 +113,14 @@ VFMT_8_8_SNORM = 53, VFMT_8_8_8_SNORM = 54, VFMT_8_8_8_8_SNORM = 55, - VFMT_10_10_10_2_UINT = 60, - VFMT_10_10_10_2_UNORM = 61, - VFMT_10_10_10_2_SINT = 62, - VFMT_10_10_10_2_SNORM = 63, + VFMT_10_10_10_2_UINT = 56, + VFMT_10_10_10_2_UNORM = 57, + VFMT_10_10_10_2_SINT = 58, + VFMT_10_10_10_2_SNORM = 59, + VFMT_2_10_10_10_UINT = 60, + VFMT_2_10_10_10_UNORM = 61, + VFMT_2_10_10_10_SINT = 62, + VFMT_2_10_10_10_SNORM = 63, }; enum a3xx_tex_fmt { @@ -124,10 +130,14 @@ TFMT_Z16_UNORM = 9, TFMT_X8Z24_UNORM = 10, TFMT_Z32_FLOAT = 11, - TFMT_NV12_UV_TILED = 17, - TFMT_NV12_Y_TILED = 19, - TFMT_NV12_UV = 21, - TFMT_NV12_Y = 23, + TFMT_UV_64X32 = 16, + TFMT_VU_64X32 = 17, + TFMT_Y_64X32 = 18, + TFMT_NV12_64X32 = 19, + TFMT_UV_LINEAR = 20, + TFMT_VU_LINEAR = 21, + TFMT_Y_LINEAR = 22, + TFMT_NV12_LINEAR = 23, TFMT_I420_Y = 24, TFMT_I420_U = 26, TFMT_I420_V = 27, @@ -138,10 +148,12 @@ TFMT_DXT1 = 36, TFMT_DXT3 = 37, TFMT_DXT5 = 38, + TFMT_2_10_10_10_UNORM = 40, TFMT_10_10_10_2_UNORM = 41, TFMT_9_9_9_E5_FLOAT = 42, TFMT_11_11_10_FLOAT = 43, TFMT_A8_UNORM = 44, + TFMT_L8_UNORM = 45, TFMT_L8_A8_UNORM = 47, TFMT_8_UNORM = 48, TFMT_8_8_UNORM = 49, @@ -183,6 +195,8 @@ TFMT_32_SINT = 92, TFMT_32_32_SINT = 93, TFMT_32_32_32_32_SINT = 95, + TFMT_2_10_10_10_UINT = 96, + TFMT_10_10_10_2_UINT = 97, TFMT_ETC2_RG11_SNORM = 112, TFMT_ETC2_RG11_UNORM = 113, TFMT_ETC2_R11_SNORM = 114, @@ -215,6 +229,9 @@ RB_R8_UINT = 14, RB_R8_SINT = 15, RB_R10G10B10A2_UNORM = 16, + RB_A2R10G10B10_UNORM = 17, + RB_R10G10B10A2_UINT = 18, + RB_A2R10G10B10_UINT = 19, RB_A8_UNORM = 20, RB_R8_UNORM = 21, RB_R16_FLOAT = 24, @@ -244,38 +261,273 @@ RB_R32G32B32A32_UINT = 59, }; +enum a3xx_cp_perfcounter_select { + CP_ALWAYS_COUNT = 0, + CP_AHB_PFPTRANS_WAIT = 3, + CP_AHB_NRTTRANS_WAIT = 6, + CP_CSF_NRT_READ_WAIT = 8, + CP_CSF_I1_FIFO_FULL = 9, + CP_CSF_I2_FIFO_FULL = 10, + CP_CSF_ST_FIFO_FULL = 11, + CP_RESERVED_12 = 12, + CP_CSF_RING_ROQ_FULL = 13, + CP_CSF_I1_ROQ_FULL = 14, + CP_CSF_I2_ROQ_FULL = 15, + CP_CSF_ST_ROQ_FULL = 16, + CP_RESERVED_17 = 17, + CP_MIU_TAG_MEM_FULL = 18, + CP_MIU_NRT_WRITE_STALLED = 22, + CP_MIU_NRT_READ_STALLED = 23, + CP_ME_REGS_RB_DONE_FIFO_FULL = 26, + CP_ME_REGS_VS_EVENT_FIFO_FULL = 27, + CP_ME_REGS_PS_EVENT_FIFO_FULL = 28, + CP_ME_REGS_CF_EVENT_FIFO_FULL = 29, + CP_ME_MICRO_RB_STARVED = 30, + CP_AHB_RBBM_DWORD_SENT = 40, + CP_ME_BUSY_CLOCKS = 41, + CP_ME_WAIT_CONTEXT_AVAIL = 42, + CP_PFP_TYPE0_PACKET = 43, + CP_PFP_TYPE3_PACKET = 44, + CP_CSF_RB_WPTR_NEQ_RPTR = 45, + CP_CSF_I1_SIZE_NEQ_ZERO = 46, + CP_CSF_I2_SIZE_NEQ_ZERO = 47, + CP_CSF_RBI1I2_FETCHING = 48, +}; + +enum a3xx_gras_tse_perfcounter_select { + GRAS_TSEPERF_INPUT_PRIM = 0, + GRAS_TSEPERF_INPUT_NULL_PRIM = 1, + GRAS_TSEPERF_TRIVAL_REJ_PRIM = 2, + GRAS_TSEPERF_CLIPPED_PRIM = 3, + GRAS_TSEPERF_NEW_PRIM = 4, + GRAS_TSEPERF_ZERO_AREA_PRIM = 5, + GRAS_TSEPERF_FACENESS_CULLED_PRIM = 6, + GRAS_TSEPERF_ZERO_PIXEL_PRIM = 7, + GRAS_TSEPERF_OUTPUT_NULL_PRIM = 8, + GRAS_TSEPERF_OUTPUT_VISIBLE_PRIM = 9, + GRAS_TSEPERF_PRE_CLIP_PRIM = 10, + GRAS_TSEPERF_POST_CLIP_PRIM = 11, + GRAS_TSEPERF_WORKING_CYCLES = 12, + GRAS_TSEPERF_PC_STARVE = 13, + GRAS_TSERASPERF_STALL = 14, +}; + +enum a3xx_gras_ras_perfcounter_select { + GRAS_RASPERF_16X16_TILES = 0, + GRAS_RASPERF_8X8_TILES = 1, + GRAS_RASPERF_4X4_TILES = 2, + GRAS_RASPERF_WORKING_CYCLES = 3, + GRAS_RASPERF_STALL_CYCLES_BY_RB = 4, + GRAS_RASPERF_STALL_CYCLES_BY_VSC = 5, + GRAS_RASPERF_STARVE_CYCLES_BY_TSE = 6, +}; + +enum a3xx_hlsq_perfcounter_select { + HLSQ_PERF_SP_VS_CONSTANT = 0, + HLSQ_PERF_SP_VS_INSTRUCTIONS = 1, + HLSQ_PERF_SP_FS_CONSTANT = 2, + HLSQ_PERF_SP_FS_INSTRUCTIONS = 3, + HLSQ_PERF_TP_STATE = 4, + HLSQ_PERF_QUADS = 5, + HLSQ_PERF_PIXELS = 6, + HLSQ_PERF_VERTICES = 7, + HLSQ_PERF_FS8_THREADS = 8, + HLSQ_PERF_FS16_THREADS = 9, + HLSQ_PERF_FS32_THREADS = 10, + HLSQ_PERF_VS8_THREADS = 11, + HLSQ_PERF_VS16_THREADS = 12, + HLSQ_PERF_SP_VS_DATA_BYTES = 13, + HLSQ_PERF_SP_FS_DATA_BYTES = 14, + HLSQ_PERF_ACTIVE_CYCLES = 15, + HLSQ_PERF_STALL_CYCLES_SP_STATE = 16, + HLSQ_PERF_STALL_CYCLES_SP_VS = 17, + HLSQ_PERF_STALL_CYCLES_SP_FS = 18, + HLSQ_PERF_STALL_CYCLES_UCHE = 19, + HLSQ_PERF_RBBM_LOAD_CYCLES = 20, + HLSQ_PERF_DI_TO_VS_START_SP0 = 21, + HLSQ_PERF_DI_TO_FS_START_SP0 = 22, + HLSQ_PERF_VS_START_TO_DONE_SP0 = 23, + HLSQ_PERF_FS_START_TO_DONE_SP0 = 24, + HLSQ_PERF_SP_STATE_COPY_CYCLES_VS = 25, + HLSQ_PERF_SP_STATE_COPY_CYCLES_FS = 26, + HLSQ_PERF_UCHE_LATENCY_CYCLES = 27, + HLSQ_PERF_UCHE_LATENCY_COUNT = 28, +}; + +enum a3xx_pc_perfcounter_select { + PC_PCPERF_VISIBILITY_STREAMS = 0, + PC_PCPERF_TOTAL_INSTANCES = 1, + PC_PCPERF_PRIMITIVES_PC_VPC = 2, + PC_PCPERF_PRIMITIVES_KILLED_BY_VS = 3, + PC_PCPERF_PRIMITIVES_VISIBLE_BY_VS = 4, + PC_PCPERF_DRAWCALLS_KILLED_BY_VS = 5, + PC_PCPERF_DRAWCALLS_VISIBLE_BY_VS = 6, + PC_PCPERF_VERTICES_TO_VFD = 7, + PC_PCPERF_REUSED_VERTICES = 8, + PC_PCPERF_CYCLES_STALLED_BY_VFD = 9, + PC_PCPERF_CYCLES_STALLED_BY_TSE = 10, + PC_PCPERF_CYCLES_STALLED_BY_VBIF = 11, + PC_PCPERF_CYCLES_IS_WORKING = 12, +}; + +enum a3xx_rb_perfcounter_select { + RB_RBPERF_ACTIVE_CYCLES_ANY = 0, + RB_RBPERF_ACTIVE_CYCLES_ALL = 1, + RB_RBPERF_STARVE_CYCLES_BY_SP = 2, + RB_RBPERF_STARVE_CYCLES_BY_RAS = 3, + RB_RBPERF_STARVE_CYCLES_BY_MARB = 4, + RB_RBPERF_STALL_CYCLES_BY_MARB = 5, + RB_RBPERF_STALL_CYCLES_BY_HLSQ = 6, + RB_RBPERF_RB_MARB_DATA = 7, + RB_RBPERF_SP_RB_QUAD = 8, + RB_RBPERF_RAS_EARLY_Z_QUADS = 9, + RB_RBPERF_GMEM_CH0_READ = 10, + RB_RBPERF_GMEM_CH1_READ = 11, + RB_RBPERF_GMEM_CH0_WRITE = 12, + RB_RBPERF_GMEM_CH1_WRITE = 13, + RB_RBPERF_CP_CONTEXT_DONE = 14, + RB_RBPERF_CP_CACHE_FLUSH = 15, + RB_RBPERF_CP_ZPASS_DONE = 16, +}; + +enum a3xx_rbbm_perfcounter_select { + RBBM_ALAWYS_ON = 0, + RBBM_VBIF_BUSY = 1, + RBBM_TSE_BUSY = 2, + RBBM_RAS_BUSY = 3, + RBBM_PC_DCALL_BUSY = 4, + RBBM_PC_VSD_BUSY = 5, + RBBM_VFD_BUSY = 6, + RBBM_VPC_BUSY = 7, + RBBM_UCHE_BUSY = 8, + RBBM_VSC_BUSY = 9, + RBBM_HLSQ_BUSY = 10, + RBBM_ANY_RB_BUSY = 11, + RBBM_ANY_TEX_BUSY = 12, + RBBM_ANY_USP_BUSY = 13, + RBBM_ANY_MARB_BUSY = 14, + RBBM_ANY_ARB_BUSY = 15, + RBBM_AHB_STATUS_BUSY = 16, + RBBM_AHB_STATUS_STALLED = 17, + RBBM_AHB_STATUS_TXFR = 18, + RBBM_AHB_STATUS_TXFR_SPLIT = 19, + RBBM_AHB_STATUS_TXFR_ERROR = 20, + RBBM_AHB_STATUS_LONG_STALL = 21, + RBBM_RBBM_STATUS_MASKED = 22, +}; + enum a3xx_sp_perfcounter_select { + SP_LM_LOAD_INSTRUCTIONS = 0, + SP_LM_STORE_INSTRUCTIONS = 1, + SP_LM_ATOMICS = 2, + SP_UCHE_LOAD_INSTRUCTIONS = 3, + SP_UCHE_STORE_INSTRUCTIONS = 4, + SP_UCHE_ATOMICS = 5, + SP_VS_TEX_INSTRUCTIONS = 6, + SP_VS_CFLOW_INSTRUCTIONS = 7, + SP_VS_EFU_INSTRUCTIONS = 8, + SP_VS_FULL_ALU_INSTRUCTIONS = 9, + SP_VS_HALF_ALU_INSTRUCTIONS = 10, + SP_FS_TEX_INSTRUCTIONS = 11, SP_FS_CFLOW_INSTRUCTIONS = 12, + SP_FS_EFU_INSTRUCTIONS = 13, SP_FS_FULL_ALU_INSTRUCTIONS = 14, - SP0_ICL1_MISSES = 26, + SP_FS_HALF_ALU_INSTRUCTIONS = 15, + SP_FS_BARY_INSTRUCTIONS = 16, + SP_VS_INSTRUCTIONS = 17, + SP_FS_INSTRUCTIONS = 18, + SP_ADDR_LOCK_COUNT = 19, + SP_UCHE_READ_TRANS = 20, + SP_UCHE_WRITE_TRANS = 21, + SP_EXPORT_VPC_TRANS = 22, + SP_EXPORT_RB_TRANS = 23, + SP_PIXELS_KILLED = 24, + SP_ICL1_REQUESTS = 25, + SP_ICL1_MISSES = 26, + SP_ICL0_REQUESTS = 27, + SP_ICL0_MISSES = 28, SP_ALU_ACTIVE_CYCLES = 29, + SP_EFU_ACTIVE_CYCLES = 30, + SP_STALL_CYCLES_BY_VPC = 31, + SP_STALL_CYCLES_BY_TP = 32, + SP_STALL_CYCLES_BY_UCHE = 33, + SP_STALL_CYCLES_BY_RB = 34, + SP_ACTIVE_CYCLES_ANY = 35, + SP_ACTIVE_CYCLES_ALL = 36, +}; + +enum a3xx_tp_perfcounter_select { + TPL1_TPPERF_L1_REQUESTS = 0, + TPL1_TPPERF_TP0_L1_REQUESTS = 1, + TPL1_TPPERF_TP0_L1_MISSES = 2, + TPL1_TPPERF_TP1_L1_REQUESTS = 3, + TPL1_TPPERF_TP1_L1_MISSES = 4, + TPL1_TPPERF_TP2_L1_REQUESTS = 5, + TPL1_TPPERF_TP2_L1_MISSES = 6, + TPL1_TPPERF_TP3_L1_REQUESTS = 7, + TPL1_TPPERF_TP3_L1_MISSES = 8, + TPL1_TPPERF_OUTPUT_TEXELS_POINT = 9, + TPL1_TPPERF_OUTPUT_TEXELS_BILINEAR = 10, + TPL1_TPPERF_OUTPUT_TEXELS_MIP = 11, + TPL1_TPPERF_OUTPUT_TEXELS_ANISO = 12, + TPL1_TPPERF_BILINEAR_OPS = 13, + TPL1_TPPERF_QUADSQUADS_OFFSET = 14, + TPL1_TPPERF_QUADQUADS_SHADOW = 15, + TPL1_TPPERF_QUADS_ARRAY = 16, + TPL1_TPPERF_QUADS_PROJECTION = 17, + TPL1_TPPERF_QUADS_GRADIENT = 18, + TPL1_TPPERF_QUADS_1D2D = 19, + TPL1_TPPERF_QUADS_3DCUBE = 20, + TPL1_TPPERF_ZERO_LOD = 21, + TPL1_TPPERF_OUTPUT_TEXELS = 22, + TPL1_TPPERF_ACTIVE_CYCLES_ANY = 23, + TPL1_TPPERF_ACTIVE_CYCLES_ALL = 24, + TPL1_TPPERF_STALL_CYCLES_BY_ARB = 25, + TPL1_TPPERF_LATENCY = 26, + TPL1_TPPERF_LATENCY_TRANS = 27, }; -enum a3xx_rop_code { - ROP_CLEAR = 0, - ROP_NOR = 1, - ROP_AND_INVERTED = 2, - ROP_COPY_INVERTED = 3, - ROP_AND_REVERSE = 4, - ROP_INVERT = 5, - ROP_XOR = 6, - ROP_NAND = 7, - ROP_AND = 8, - ROP_EQUIV = 9, - ROP_NOOP = 10, - ROP_OR_INVERTED = 11, - ROP_COPY = 12, - ROP_OR_REVERSE = 13, - ROP_OR = 14, - ROP_SET = 15, -}; - -enum a3xx_rb_blend_opcode { - BLEND_DST_PLUS_SRC = 0, - BLEND_SRC_MINUS_DST = 1, - BLEND_DST_MINUS_SRC = 2, - BLEND_MIN_DST_SRC = 3, - BLEND_MAX_DST_SRC = 4, +enum a3xx_vfd_perfcounter_select { + VFD_PERF_UCHE_BYTE_FETCHED = 0, + VFD_PERF_UCHE_TRANS = 1, + VFD_PERF_VPC_BYPASS_COMPONENTS = 2, + VFD_PERF_FETCH_INSTRUCTIONS = 3, + VFD_PERF_DECODE_INSTRUCTIONS = 4, + VFD_PERF_ACTIVE_CYCLES = 5, + VFD_PERF_STALL_CYCLES_UCHE = 6, + VFD_PERF_STALL_CYCLES_HLSQ = 7, + VFD_PERF_STALL_CYCLES_VPC_BYPASS = 8, + VFD_PERF_STALL_CYCLES_VPC_ALLOC = 9, +}; + +enum a3xx_vpc_perfcounter_select { + VPC_PERF_SP_LM_PRIMITIVES = 0, + VPC_PERF_COMPONENTS_FROM_SP = 1, + VPC_PERF_SP_LM_COMPONENTS = 2, + VPC_PERF_ACTIVE_CYCLES = 3, + VPC_PERF_STALL_CYCLES_LM = 4, + VPC_PERF_STALL_CYCLES_RAS = 5, +}; + +enum a3xx_uche_perfcounter_select { + UCHE_UCHEPERF_VBIF_READ_BEATS_TP = 0, + UCHE_UCHEPERF_VBIF_READ_BEATS_VFD = 1, + UCHE_UCHEPERF_VBIF_READ_BEATS_HLSQ = 2, + UCHE_UCHEPERF_VBIF_READ_BEATS_MARB = 3, + UCHE_UCHEPERF_VBIF_READ_BEATS_SP = 4, + UCHE_UCHEPERF_READ_REQUESTS_TP = 8, + UCHE_UCHEPERF_READ_REQUESTS_VFD = 9, + UCHE_UCHEPERF_READ_REQUESTS_HLSQ = 10, + UCHE_UCHEPERF_READ_REQUESTS_MARB = 11, + UCHE_UCHEPERF_READ_REQUESTS_SP = 12, + UCHE_UCHEPERF_WRITE_REQUESTS_MARB = 13, + UCHE_UCHEPERF_WRITE_REQUESTS_SP = 14, + UCHE_UCHEPERF_TAG_CHECK_FAILS = 15, + UCHE_UCHEPERF_EVICTS = 16, + UCHE_UCHEPERF_FLUSHES = 17, + UCHE_UCHEPERF_VBIF_LATENCY_CYCLES = 18, + UCHE_UCHEPERF_VBIF_LATENCY_SAMPLES = 19, + UCHE_UCHEPERF_ACTIVE_CYCLES = 20, }; enum a3xx_intp_mode { @@ -1138,13 +1390,14 @@ { return ((val) << A3XX_RB_COPY_CONTROL_MODE__SHIFT) & A3XX_RB_COPY_CONTROL_MODE__MASK; } +#define A3XX_RB_COPY_CONTROL_MSAA_SRGB_DOWNSAMPLE 0x00000080 #define A3XX_RB_COPY_CONTROL_FASTCLEAR__MASK 0x00000f00 #define A3XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT 8 static inline uint32_t A3XX_RB_COPY_CONTROL_FASTCLEAR(uint32_t val) { return ((val) << A3XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT) & A3XX_RB_COPY_CONTROL_FASTCLEAR__MASK; } -#define A3XX_RB_COPY_CONTROL_UNK12 0x00001000 +#define A3XX_RB_COPY_CONTROL_DEPTH32_RESOLVE 0x00001000 #define A3XX_RB_COPY_CONTROL_GMEM_BASE__MASK 0xffffc000 #define A3XX_RB_COPY_CONTROL_GMEM_BASE__SHIFT 14 static inline uint32_t A3XX_RB_COPY_CONTROL_GMEM_BASE(uint32_t val) @@ -1217,7 +1470,7 @@ { return ((val) << A3XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT) & A3XX_RB_DEPTH_CONTROL_ZFUNC__MASK; } -#define A3XX_RB_DEPTH_CONTROL_BF_ENABLE 0x00000080 +#define A3XX_RB_DEPTH_CONTROL_Z_CLAMP_ENABLE 0x00000080 #define A3XX_RB_DEPTH_CONTROL_Z_TEST_ENABLE 0x80000000 #define REG_A3XX_RB_DEPTH_CLEAR 0x00002101 @@ -1429,15 +1682,23 @@ #define REG_A3XX_PC_RESTART_INDEX 0x000021ed #define REG_A3XX_HLSQ_CONTROL_0_REG 0x00002200 -#define A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK 0x00000010 +#define A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK 0x00000030 #define A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT 4 static inline uint32_t A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE(enum a3xx_threadsize val) { return ((val) << A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT) & A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK; } #define A3XX_HLSQ_CONTROL_0_REG_FSSUPERTHREADENABLE 0x00000040 +#define A3XX_HLSQ_CONTROL_0_REG_COMPUTEMODE 0x00000100 #define A3XX_HLSQ_CONTROL_0_REG_SPSHADERRESTART 0x00000200 #define A3XX_HLSQ_CONTROL_0_REG_RESERVED2 0x00000400 +#define A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__MASK 0x00fff000 +#define A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__SHIFT 12 +static inline uint32_t A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC(uint32_t val) +{ + return ((val) << A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__SHIFT) & A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__MASK; +} +#define A3XX_HLSQ_CONTROL_0_REG_FSONLYTEX 0x02000000 #define A3XX_HLSQ_CONTROL_0_REG_CHUNKDISABLE 0x04000000 #define A3XX_HLSQ_CONTROL_0_REG_CONSTMODE__MASK 0x08000000 #define A3XX_HLSQ_CONTROL_0_REG_CONSTMODE__SHIFT 27 @@ -1451,17 +1712,39 @@ #define A3XX_HLSQ_CONTROL_0_REG_SINGLECONTEXT 0x80000000 #define REG_A3XX_HLSQ_CONTROL_1_REG 0x00002201 -#define A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__MASK 0x00000040 +#define A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__MASK 0x000000c0 #define A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__SHIFT 6 static inline uint32_t A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE(enum a3xx_threadsize val) { return ((val) << A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__SHIFT) & A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__MASK; } #define A3XX_HLSQ_CONTROL_1_REG_VSSUPERTHREADENABLE 0x00000100 -#define A3XX_HLSQ_CONTROL_1_REG_RESERVED1 0x00000200 -#define A3XX_HLSQ_CONTROL_1_REG_ZWCOORD 0x02000000 +#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__MASK 0x00ff0000 +#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__SHIFT 16 +static inline uint32_t A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID(uint32_t val) +{ + return ((val) << A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__SHIFT) & A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__MASK; +} +#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__MASK 0xff000000 +#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__SHIFT 24 +static inline uint32_t A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID(uint32_t val) +{ + return ((val) << A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__SHIFT) & A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__MASK; +} #define REG_A3XX_HLSQ_CONTROL_2_REG 0x00002202 +#define A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__MASK 0x000003fc +#define A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__SHIFT 2 +static inline uint32_t A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID(uint32_t val) +{ + return ((val) << A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__SHIFT) & A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__MASK; +} +#define A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__MASK 0x03fc0000 +#define A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__SHIFT 18 +static inline uint32_t A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID(uint32_t val) +{ + return ((val) << A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__SHIFT) & A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__MASK; +} #define A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__MASK 0xfc000000 #define A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__SHIFT 26 static inline uint32_t A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD(uint32_t val) @@ -1478,13 +1761,13 @@ } #define REG_A3XX_HLSQ_VS_CONTROL_REG 0x00002204 -#define A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK 0x00000fff +#define A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK 0x000003ff #define A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT 0 static inline uint32_t A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH(uint32_t val) { return ((val) << A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT) & A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK; } -#define A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET__MASK 0x00fff000 +#define A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET__MASK 0x001ff000 #define A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET__SHIFT 12 static inline uint32_t A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET(uint32_t val) { @@ -1498,13 +1781,13 @@ } #define REG_A3XX_HLSQ_FS_CONTROL_REG 0x00002205 -#define A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK 0x00000fff +#define A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK 0x000003ff #define A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT 0 static inline uint32_t A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH(uint32_t val) { return ((val) << A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT) & A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK; } -#define A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET__MASK 0x00fff000 +#define A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET__MASK 0x001ff000 #define A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET__SHIFT 12 static inline uint32_t A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET(uint32_t val) { @@ -1518,13 +1801,13 @@ } #define REG_A3XX_HLSQ_CONST_VSPRESV_RANGE_REG 0x00002206 -#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__MASK 0x0000ffff +#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__MASK 0x000001ff #define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__SHIFT 0 static inline uint32_t A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY(uint32_t val) { return ((val) << A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__SHIFT) & A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__MASK; } -#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY__MASK 0xffff0000 +#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY__MASK 0x01ff0000 #define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY__SHIFT 16 static inline uint32_t A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY(uint32_t val) { @@ -1532,13 +1815,13 @@ } #define REG_A3XX_HLSQ_CONST_FSPRESV_RANGE_REG 0x00002207 -#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__MASK 0x0000ffff +#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__MASK 0x000001ff #define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__SHIFT 0 static inline uint32_t A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY(uint32_t val) { return ((val) << A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__SHIFT) & A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__MASK; } -#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY__MASK 0xffff0000 +#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY__MASK 0x01ff0000 #define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY__SHIFT 16 static inline uint32_t A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY(uint32_t val) { @@ -1620,12 +1903,24 @@ } #define REG_A3XX_VFD_CONTROL_1 0x00002241 -#define A3XX_VFD_CONTROL_1_MAXSTORAGE__MASK 0x0000ffff +#define A3XX_VFD_CONTROL_1_MAXSTORAGE__MASK 0x0000000f #define A3XX_VFD_CONTROL_1_MAXSTORAGE__SHIFT 0 static inline uint32_t A3XX_VFD_CONTROL_1_MAXSTORAGE(uint32_t val) { return ((val) << A3XX_VFD_CONTROL_1_MAXSTORAGE__SHIFT) & A3XX_VFD_CONTROL_1_MAXSTORAGE__MASK; } +#define A3XX_VFD_CONTROL_1_MAXTHRESHOLD__MASK 0x000000f0 +#define A3XX_VFD_CONTROL_1_MAXTHRESHOLD__SHIFT 4 +static inline uint32_t A3XX_VFD_CONTROL_1_MAXTHRESHOLD(uint32_t val) +{ + return ((val) << A3XX_VFD_CONTROL_1_MAXTHRESHOLD__SHIFT) & A3XX_VFD_CONTROL_1_MAXTHRESHOLD__MASK; +} +#define A3XX_VFD_CONTROL_1_MINTHRESHOLD__MASK 0x00000f00 +#define A3XX_VFD_CONTROL_1_MINTHRESHOLD__SHIFT 8 +static inline uint32_t A3XX_VFD_CONTROL_1_MINTHRESHOLD(uint32_t val) +{ + return ((val) << A3XX_VFD_CONTROL_1_MINTHRESHOLD__SHIFT) & A3XX_VFD_CONTROL_1_MINTHRESHOLD__MASK; +} #define A3XX_VFD_CONTROL_1_REGID4VTX__MASK 0x00ff0000 #define A3XX_VFD_CONTROL_1_REGID4VTX__SHIFT 16 static inline uint32_t A3XX_VFD_CONTROL_1_REGID4VTX(uint32_t val) @@ -2008,24 +2303,19 @@ return ((val) << A3XX_SP_VS_CTRL_REG0_INSTRBUFFERMODE__SHIFT) & A3XX_SP_VS_CTRL_REG0_INSTRBUFFERMODE__MASK; } #define A3XX_SP_VS_CTRL_REG0_CACHEINVALID 0x00000004 +#define A3XX_SP_VS_CTRL_REG0_ALUSCHMODE 0x00000008 #define A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x000003f0 #define A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 4 static inline uint32_t A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) { return ((val) << A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK; } -#define A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0003fc00 +#define A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0000fc00 #define A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 10 static inline uint32_t A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) { return ((val) << A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK; } -#define A3XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__MASK 0x000c0000 -#define A3XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__SHIFT 18 -static inline uint32_t A3XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP(uint32_t val) -{ - return ((val) << A3XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__SHIFT) & A3XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__MASK; -} #define A3XX_SP_VS_CTRL_REG0_THREADSIZE__MASK 0x00100000 #define A3XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT 20 static inline uint32_t A3XX_SP_VS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) @@ -2033,8 +2323,6 @@ return ((val) << A3XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT) & A3XX_SP_VS_CTRL_REG0_THREADSIZE__MASK; } #define A3XX_SP_VS_CTRL_REG0_SUPERTHREADMODE 0x00200000 -#define A3XX_SP_VS_CTRL_REG0_PIXLODENABLE 0x00400000 -#define A3XX_SP_VS_CTRL_REG0_COMPUTEMODE 0x00800000 #define A3XX_SP_VS_CTRL_REG0_LENGTH__MASK 0xff000000 #define A3XX_SP_VS_CTRL_REG0_LENGTH__SHIFT 24 static inline uint32_t A3XX_SP_VS_CTRL_REG0_LENGTH(uint32_t val) @@ -2075,7 +2363,8 @@ { return ((val) << A3XX_SP_VS_PARAM_REG_PSIZEREGID__SHIFT) & A3XX_SP_VS_PARAM_REG_PSIZEREGID__MASK; } -#define A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__MASK 0xfff00000 +#define A3XX_SP_VS_PARAM_REG_POS2DMODE 0x00010000 +#define A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__MASK 0x01f00000 #define A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__SHIFT 20 static inline uint32_t A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR(uint32_t val) { @@ -2085,24 +2374,26 @@ static inline uint32_t REG_A3XX_SP_VS_OUT(uint32_t i0) { return 0x000022c7 + 0x1*i0; } static inline uint32_t REG_A3XX_SP_VS_OUT_REG(uint32_t i0) { return 0x000022c7 + 0x1*i0; } -#define A3XX_SP_VS_OUT_REG_A_REGID__MASK 0x000001ff +#define A3XX_SP_VS_OUT_REG_A_REGID__MASK 0x000000ff #define A3XX_SP_VS_OUT_REG_A_REGID__SHIFT 0 static inline uint32_t A3XX_SP_VS_OUT_REG_A_REGID(uint32_t val) { return ((val) << A3XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A3XX_SP_VS_OUT_REG_A_REGID__MASK; } +#define A3XX_SP_VS_OUT_REG_A_HALF 0x00000100 #define A3XX_SP_VS_OUT_REG_A_COMPMASK__MASK 0x00001e00 #define A3XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT 9 static inline uint32_t A3XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val) { return ((val) << A3XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A3XX_SP_VS_OUT_REG_A_COMPMASK__MASK; } -#define A3XX_SP_VS_OUT_REG_B_REGID__MASK 0x01ff0000 +#define A3XX_SP_VS_OUT_REG_B_REGID__MASK 0x00ff0000 #define A3XX_SP_VS_OUT_REG_B_REGID__SHIFT 16 static inline uint32_t A3XX_SP_VS_OUT_REG_B_REGID(uint32_t val) { return ((val) << A3XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A3XX_SP_VS_OUT_REG_B_REGID__MASK; } +#define A3XX_SP_VS_OUT_REG_B_HALF 0x01000000 #define A3XX_SP_VS_OUT_REG_B_COMPMASK__MASK 0x1e000000 #define A3XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT 25 static inline uint32_t A3XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val) @@ -2113,25 +2404,25 @@ static inline uint32_t REG_A3XX_SP_VS_VPC_DST(uint32_t i0) { return 0x000022d0 + 0x1*i0; } static inline uint32_t REG_A3XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x000022d0 + 0x1*i0; } -#define A3XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK 0x000000ff +#define A3XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK 0x0000007f #define A3XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT 0 static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val) { return ((val) << A3XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A3XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK; } -#define A3XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK 0x0000ff00 +#define A3XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK 0x00007f00 #define A3XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT 8 static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val) { return ((val) << A3XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A3XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK; } -#define A3XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK 0x00ff0000 +#define A3XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK 0x007f0000 #define A3XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT 16 static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val) { return ((val) << A3XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A3XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK; } -#define A3XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK 0xff000000 +#define A3XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK 0x7f000000 #define A3XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT 24 static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val) { @@ -2139,6 +2430,12 @@ } #define REG_A3XX_SP_VS_OBJ_OFFSET_REG 0x000022d4 +#define A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK 0x0000ffff +#define A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT 0 +static inline uint32_t A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET(uint32_t val) +{ + return ((val) << A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT) & A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK; +} #define A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK 0x01ff0000 #define A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT 16 static inline uint32_t A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val) @@ -2155,8 +2452,38 @@ #define REG_A3XX_SP_VS_OBJ_START_REG 0x000022d5 #define REG_A3XX_SP_VS_PVT_MEM_PARAM_REG 0x000022d6 +#define A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK 0x000000ff +#define A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT 0 +static inline uint32_t A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM(uint32_t val) +{ + return ((val) << A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT) & A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK; +} +#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK 0x00ffff00 +#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT 8 +static inline uint32_t A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET(uint32_t val) +{ + return ((val) << A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT) & A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK; +} +#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK 0xff000000 +#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT 24 +static inline uint32_t A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD(uint32_t val) +{ + return ((val) << A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT) & A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK; +} #define REG_A3XX_SP_VS_PVT_MEM_ADDR_REG 0x000022d7 +#define A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__MASK 0x0000001f +#define A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT 0 +static inline uint32_t A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN(uint32_t val) +{ + return ((val) << A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT) & A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__MASK; +} +#define A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK 0xffffffe0 +#define A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT 5 +static inline uint32_t A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS(uint32_t val) +{ + return ((val >> 5) << A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT) & A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK; +} #define REG_A3XX_SP_VS_PVT_MEM_SIZE_REG 0x000022d8 @@ -2182,24 +2509,22 @@ return ((val) << A3XX_SP_FS_CTRL_REG0_INSTRBUFFERMODE__SHIFT) & A3XX_SP_FS_CTRL_REG0_INSTRBUFFERMODE__MASK; } #define A3XX_SP_FS_CTRL_REG0_CACHEINVALID 0x00000004 +#define A3XX_SP_FS_CTRL_REG0_ALUSCHMODE 0x00000008 #define A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x000003f0 #define A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 4 static inline uint32_t A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) { return ((val) << A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK; } -#define A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0003fc00 +#define A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0000fc00 #define A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 10 static inline uint32_t A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) { return ((val) << A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK; } -#define A3XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__MASK 0x000c0000 -#define A3XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__SHIFT 18 -static inline uint32_t A3XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP(uint32_t val) -{ - return ((val) << A3XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__SHIFT) & A3XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__MASK; -} +#define A3XX_SP_FS_CTRL_REG0_FSBYPASSENABLE 0x00020000 +#define A3XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP 0x00040000 +#define A3XX_SP_FS_CTRL_REG0_OUTORDERED 0x00080000 #define A3XX_SP_FS_CTRL_REG0_THREADSIZE__MASK 0x00100000 #define A3XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT 20 static inline uint32_t A3XX_SP_FS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) @@ -2235,7 +2560,7 @@ { return ((val) << A3XX_SP_FS_CTRL_REG1_INITIALOUTSTANDING__SHIFT) & A3XX_SP_FS_CTRL_REG1_INITIALOUTSTANDING__MASK; } -#define A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET__MASK 0x3f000000 +#define A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET__MASK 0x7f000000 #define A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET__SHIFT 24 static inline uint32_t A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET(uint32_t val) { @@ -2243,6 +2568,12 @@ } #define REG_A3XX_SP_FS_OBJ_OFFSET_REG 0x000022e2 +#define A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK 0x0000ffff +#define A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT 0 +static inline uint32_t A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET(uint32_t val) +{ + return ((val) << A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT) & A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK; +} #define A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK 0x01ff0000 #define A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT 16 static inline uint32_t A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val) @@ -2259,8 +2590,38 @@ #define REG_A3XX_SP_FS_OBJ_START_REG 0x000022e3 #define REG_A3XX_SP_FS_PVT_MEM_PARAM_REG 0x000022e4 +#define A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK 0x000000ff +#define A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT 0 +static inline uint32_t A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM(uint32_t val) +{ + return ((val) << A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT) & A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK; +} +#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK 0x00ffff00 +#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT 8 +static inline uint32_t A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET(uint32_t val) +{ + return ((val) << A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT) & A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK; +} +#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK 0xff000000 +#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT 24 +static inline uint32_t A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD(uint32_t val) +{ + return ((val) << A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT) & A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK; +} #define REG_A3XX_SP_FS_PVT_MEM_ADDR_REG 0x000022e5 +#define A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__MASK 0x0000001f +#define A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT 0 +static inline uint32_t A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN(uint32_t val) +{ + return ((val) << A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT) & A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__MASK; +} +#define A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK 0xffffffe0 +#define A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT 5 +static inline uint32_t A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS(uint32_t val) +{ + return ((val >> 5) << A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT) & A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK; +} #define REG_A3XX_SP_FS_PVT_MEM_SIZE_REG 0x000022e6 diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/a4xx_gpu.c linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a4xx_gpu.c --- linux-4.4.115/drivers/gpu/drm/msm/adreno/a4xx_gpu.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a4xx_gpu.c 2019-01-22 16:16:23.483246225 +0100 @@ -1,4 +1,4 @@ -/* Copyright (c) 2014 The Linux Foundation. All rights reserved. +/* Copyright (c) 2014,2018 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -31,6 +31,7 @@ extern bool hang_debug; static void a4xx_dump(struct msm_gpu *gpu); +static bool a4xx_idle(struct msm_gpu *gpu); /* * a4xx_enable_hwcg() - Program the clock control registers @@ -102,14 +103,20 @@ gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00000222); gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL_HLSQ , 0x00000000); gpu_write(gpu, REG_A4XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000); - gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_HLSQ, 0x00020000); + gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_HLSQ, 0x00220000); + /* Early A430's have a timing issue with SP/TP power collapse; + disabling HW clock gating prevents it. */ + if (adreno_is_a430(adreno_gpu) && adreno_gpu->rev.patchid < 2) + gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL, 0); + else gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL, 0xAAAAAAAA); gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL2, 0); } -static void a4xx_me_init(struct msm_gpu *gpu) + +static bool a4xx_me_init(struct msm_gpu *gpu) { - struct msm_ringbuffer *ring = gpu->rb; + struct msm_ringbuffer *ring = gpu->rb[0]; OUT_PKT3(ring, CP_ME_INIT, 17); OUT_RING(ring, 0x000003f7); @@ -130,8 +137,8 @@ OUT_RING(ring, 0x00000000); OUT_RING(ring, 0x00000000); - gpu->funcs->flush(gpu); - gpu->funcs->idle(gpu); + gpu->funcs->flush(gpu, ring); + return a4xx_idle(gpu); } static int a4xx_hw_init(struct msm_gpu *gpu) @@ -141,7 +148,7 @@ uint32_t *ptr, len; int i, ret; - if (adreno_is_a4xx(adreno_gpu)) { + if (adreno_is_a420(adreno_gpu)) { gpu_write(gpu, REG_A4XX_VBIF_ABIT_SORT, 0x0001001F); gpu_write(gpu, REG_A4XX_VBIF_ABIT_SORT_CONF, 0x000000A4); gpu_write(gpu, REG_A4XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000001); @@ -150,6 +157,13 @@ gpu_write(gpu, REG_A4XX_VBIF_IN_WR_LIM_CONF0, 0x18181818); gpu_write(gpu, REG_A4XX_VBIF_IN_WR_LIM_CONF1, 0x00000018); gpu_write(gpu, REG_A4XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); + } else if (adreno_is_a430(adreno_gpu)) { + gpu_write(gpu, REG_A4XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000001); + gpu_write(gpu, REG_A4XX_VBIF_IN_RD_LIM_CONF0, 0x18181818); + gpu_write(gpu, REG_A4XX_VBIF_IN_RD_LIM_CONF1, 0x00000018); + gpu_write(gpu, REG_A4XX_VBIF_IN_WR_LIM_CONF0, 0x18181818); + gpu_write(gpu, REG_A4XX_VBIF_IN_WR_LIM_CONF1, 0x00000018); + gpu_write(gpu, REG_A4XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003); } else { BUG(); } @@ -161,6 +175,10 @@ gpu_write(gpu, REG_A4XX_RBBM_SP_HYST_CNT, 0x10); gpu_write(gpu, REG_A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL, 0x10); + if (adreno_is_a430(adreno_gpu)) { + gpu_write(gpu, REG_A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL2, 0x30); + } + /* Enable the RBBM error reporting bits */ gpu_write(gpu, REG_A4XX_RBBM_AHB_CTL0, 0x00000001); @@ -183,6 +201,14 @@ /* Turn on performance counters: */ gpu_write(gpu, REG_A4XX_RBBM_PERFCTR_CTL, 0x01); + /* use the first CP counter for timestamp queries.. userspace may set + * this as well but it selects the same counter/countable: + */ + gpu_write(gpu, REG_A4XX_CP_PERFCTR_CP_SEL_0, CP_ALWAYS_COUNT); + + if (adreno_is_a430(adreno_gpu)) + gpu_write(gpu, REG_A4XX_UCHE_CACHE_WAYS_VFD, 0x07); + /* Disable L2 bypass to avoid UCHE out of bounds errors */ gpu_write(gpu, REG_A4XX_UCHE_TRAP_BASE_LO, 0xffff0000); gpu_write(gpu, REG_A4XX_UCHE_TRAP_BASE_HI, 0xffff0000); @@ -190,6 +216,15 @@ gpu_write(gpu, REG_A4XX_CP_DEBUG, (1 << 25) | (adreno_is_a420(adreno_gpu) ? (1 << 29) : 0)); + /* On A430 enable SP regfile sleep for power savings */ + /* TODO downstream does this for !420, so maybe applies for 405 too? */ + if (!adreno_is_a420(adreno_gpu)) { + gpu_write(gpu, REG_A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_0, + 0x00000441); + gpu_write(gpu, REG_A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_1, + 0x00000441); + } + a4xx_enable_hwcg(gpu); /* @@ -204,10 +239,6 @@ gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_HLSQ, val); } - ret = adreno_hw_init(gpu); - if (ret) - return ret; - /* setup access protection: */ gpu_write(gpu, REG_A4XX_CP_PROTECT_CTRL, 0x00000007); @@ -262,8 +293,7 @@ /* clear ME_HALT to start micro engine */ gpu_write(gpu, REG_A4XX_CP_ME_CNTL, 0); - a4xx_me_init(gpu); - return 0; + return a4xx_me_init(gpu) ? 0 : -EINVAL; } static void a4xx_recover(struct msm_gpu *gpu) @@ -297,17 +327,21 @@ kfree(a4xx_gpu); } -static void a4xx_idle(struct msm_gpu *gpu) +static bool a4xx_idle(struct msm_gpu *gpu) { /* wait for ringbuffer to drain: */ - adreno_idle(gpu); + if (!adreno_idle(gpu, gpu->rb[0])) + return false; /* then wait for GPU to finish: */ if (spin_until(!(gpu_read(gpu, REG_A4XX_RBBM_STATUS) & - A4XX_RBBM_STATUS_GPU_BUSY))) + A4XX_RBBM_STATUS_GPU_BUSY))) { DRM_ERROR("%s: timeout waiting for GPU to idle!\n", gpu->name); - /* TODO maybe we need to reset GPU here to recover from hang? */ + return false; + } + + return true; } static irqreturn_t a4xx_irq(struct msm_gpu *gpu) @@ -409,12 +443,8 @@ #ifdef CONFIG_DEBUG_FS static void a4xx_show(struct msm_gpu *gpu, struct seq_file *m) { - gpu->funcs->pm_resume(gpu); - seq_printf(m, "status: %08x\n", gpu_read(gpu, REG_A4XX_RBBM_STATUS)); - gpu->funcs->pm_suspend(gpu); - adreno_show(gpu, m); } @@ -422,87 +452,13 @@ /* Register offset defines for A4XX, in order of enum adreno_regs */ static const unsigned int a4xx_register_offsets[REG_ADRENO_REGISTER_MAX] = { - REG_ADRENO_DEFINE(REG_ADRENO_CP_DEBUG, REG_A4XX_CP_DEBUG), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_WADDR, REG_A4XX_CP_ME_RAM_WADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_DATA, REG_A4XX_CP_ME_RAM_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_DATA, - REG_A4XX_CP_PFP_UCODE_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_ADDR, - REG_A4XX_CP_PFP_UCODE_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_WFI_PEND_CTR, REG_A4XX_CP_WFI_PEND_CTR), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A4XX_CP_RB_BASE), + REG_ADRENO_SKIP(REG_ADRENO_CP_RB_BASE_HI), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A4XX_CP_RB_RPTR_ADDR), + REG_ADRENO_SKIP(REG_ADRENO_CP_RB_RPTR_ADDR_HI), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A4XX_CP_RB_RPTR), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A4XX_CP_RB_WPTR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_CTRL, REG_A4XX_CP_PROTECT_CTRL), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_CNTL, REG_A4XX_CP_ME_CNTL), REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A4XX_CP_RB_CNTL), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BASE, REG_A4XX_CP_IB1_BASE), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BUFSZ, REG_A4XX_CP_IB1_BUFSZ), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BASE, REG_A4XX_CP_IB2_BASE), - REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BUFSZ, REG_A4XX_CP_IB2_BUFSZ), - REG_ADRENO_DEFINE(REG_ADRENO_CP_TIMESTAMP, REG_AXXX_CP_SCRATCH_REG0), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_RADDR, REG_A4XX_CP_ME_RAM_RADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_ADDR, REG_A4XX_CP_ROQ_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_DATA, REG_A4XX_CP_ROQ_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_ADDR, REG_A4XX_CP_MERCIU_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA, REG_A4XX_CP_MERCIU_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA2, REG_A4XX_CP_MERCIU_DATA2), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_ADDR, REG_A4XX_CP_MEQ_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_DATA, REG_A4XX_CP_MEQ_DATA), - REG_ADRENO_DEFINE(REG_ADRENO_CP_HW_FAULT, REG_A4XX_CP_HW_FAULT), - REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_STATUS, - REG_A4XX_CP_PROTECT_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_ADDR, REG_A4XX_CP_SCRATCH_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_UMSK, REG_A4XX_CP_SCRATCH_UMASK), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_STATUS, REG_A4XX_RBBM_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_CTL, - REG_A4XX_RBBM_PERFCTR_CTL), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD0, - REG_A4XX_RBBM_PERFCTR_LOAD_CMD0), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD1, - REG_A4XX_RBBM_PERFCTR_LOAD_CMD1), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD2, - REG_A4XX_RBBM_PERFCTR_LOAD_CMD2), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_PWR_1_LO, - REG_A4XX_RBBM_PERFCTR_PWR_1_LO), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_MASK, REG_A4XX_RBBM_INT_0_MASK), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_STATUS, - REG_A4XX_RBBM_INT_0_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_ERROR_STATUS, - REG_A4XX_RBBM_AHB_ERROR_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_CMD, REG_A4XX_RBBM_AHB_CMD), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_CLOCK_CTL, REG_A4XX_RBBM_CLOCK_CTL), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_ME_SPLIT_STATUS, - REG_A4XX_RBBM_AHB_ME_SPLIT_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_PFP_SPLIT_STATUS, - REG_A4XX_RBBM_AHB_PFP_SPLIT_STATUS), - REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_SEL, - REG_A4XX_VPC_DEBUG_RAM_SEL), - REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_READ, - REG_A4XX_VPC_DEBUG_RAM_READ), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_CLEAR_CMD, - REG_A4XX_RBBM_INT_CLEAR_CMD), - REG_ADRENO_DEFINE(REG_ADRENO_VSC_SIZE_ADDRESS, - REG_A4XX_VSC_SIZE_ADDRESS), - REG_ADRENO_DEFINE(REG_ADRENO_VFD_CONTROL_0, REG_A4XX_VFD_CONTROL_0), - REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_PVT_MEM_ADDR_REG, - REG_A4XX_SP_VS_PVT_MEM_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_PVT_MEM_ADDR_REG, - REG_A4XX_SP_FS_PVT_MEM_ADDR), - REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_OBJ_START_REG, - REG_A4XX_SP_VS_OBJ_START), - REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_OBJ_START_REG, - REG_A4XX_SP_FS_OBJ_START), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_RBBM_CTL, REG_A4XX_RBBM_RBBM_CTL), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_SW_RESET_CMD, - REG_A4XX_RBBM_SW_RESET_CMD), - REG_ADRENO_DEFINE(REG_ADRENO_UCHE_INVALIDATE0, - REG_A4XX_UCHE_INVALIDATE0), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_LO, - REG_A4XX_RBBM_PERFCTR_LOAD_VALUE_LO), - REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_HI, - REG_A4XX_RBBM_PERFCTR_LOAD_VALUE_HI), }; static void a4xx_dump(struct msm_gpu *gpu) @@ -512,23 +468,67 @@ adreno_dump(gpu); } +static int a4xx_pm_resume(struct msm_gpu *gpu) { + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + int ret; + + ret = msm_gpu_pm_resume(gpu); + if (ret) + return ret; + + if (adreno_is_a430(adreno_gpu)) { + unsigned int reg; + /* Set the default register values; set SW_COLLAPSE to 0 */ + gpu_write(gpu, REG_A4XX_RBBM_POWER_CNTL_IP, 0x778000); + do { + udelay(5); + reg = gpu_read(gpu, REG_A4XX_RBBM_POWER_STATUS); + } while (!(reg & A4XX_RBBM_POWER_CNTL_IP_SP_TP_PWR_ON)); + } + return 0; +} + +static int a4xx_pm_suspend(struct msm_gpu *gpu) { + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + int ret; + + ret = msm_gpu_pm_suspend(gpu); + if (ret) + return ret; + + if (adreno_is_a430(adreno_gpu)) { + /* Set the default register values; set SW_COLLAPSE to 1 */ + gpu_write(gpu, REG_A4XX_RBBM_POWER_CNTL_IP, 0x778001); + } + return 0; +} + +static int a4xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) +{ + *value = gpu_read64(gpu, REG_A4XX_RBBM_PERFCTR_CP_0_LO, + REG_A4XX_RBBM_PERFCTR_CP_0_HI); + + return 0; +} + static const struct adreno_gpu_funcs funcs = { .base = { .get_param = adreno_get_param, .hw_init = a4xx_hw_init, - .pm_suspend = msm_gpu_pm_suspend, - .pm_resume = msm_gpu_pm_resume, + .pm_suspend = a4xx_pm_suspend, + .pm_resume = a4xx_pm_resume, .recover = a4xx_recover, - .last_fence = adreno_last_fence, + .submitted_fence = adreno_submitted_fence, .submit = adreno_submit, .flush = adreno_flush, - .idle = a4xx_idle, + .active_ring = adreno_active_ring, .irq = a4xx_irq, .destroy = a4xx_destroy, #ifdef CONFIG_DEBUG_FS .show = a4xx_show, #endif }, + .get_timestamp = a4xx_get_timestamp, }; struct msm_gpu *a4xx_gpu_init(struct drm_device *dev) @@ -538,6 +538,7 @@ struct msm_gpu *gpu; struct msm_drm_private *priv = dev->dev_private; struct platform_device *pdev = priv->gpu_pdev; + struct msm_gpu_config a4xx_config = { 0 }; int ret; if (!pdev) { @@ -555,15 +556,19 @@ adreno_gpu = &a4xx_gpu->base; gpu = &adreno_gpu->base; - a4xx_gpu->pdev = pdev; - gpu->perfcntrs = NULL; gpu->num_perfcntrs = 0; adreno_gpu->registers = a4xx_registers; adreno_gpu->reg_offsets = a4xx_register_offsets; - ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs); + a4xx_config.ioname = MSM_GPU_DEFAULT_IONAME; + a4xx_config.irqname = MSM_GPU_DEFAULT_IRQNAME; + a4xx_config.nr_rings = 1; + a4xx_config.va_start = 0x300000; + a4xx_config.va_end = 0xffffffff; + + ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, &a4xx_config); if (ret) goto fail; @@ -582,7 +587,7 @@ #endif } - if (!gpu->mmu) { + if (!gpu->aspace) { /* TODO we think it is possible to configure the GPU to * restrict access to VRAM carveout. But the required * registers are unknown. For now just bail out and diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/a4xx_gpu.h linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a4xx_gpu.h --- linux-4.4.115/drivers/gpu/drm/msm/adreno/a4xx_gpu.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a4xx_gpu.h 2019-01-22 16:16:23.483246225 +0100 @@ -1,4 +1,4 @@ -/* Copyright (c) 2014 The Linux Foundation. All rights reserved. +/* Copyright (c) 2014,2018 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -23,7 +23,6 @@ struct a4xx_gpu { struct adreno_gpu base; - struct platform_device *pdev; /* if OCMEM is used for GMEM: */ uint32_t ocmem_base; diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/a4xx.xml.h linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a4xx.xml.h --- linux-4.4.115/drivers/gpu/drm/msm/adreno/a4xx.xml.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/a4xx.xml.h 2019-01-22 16:16:23.479246189 +0100 @@ -8,17 +8,19 @@ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00) +- ./adreno.xml ( 431 bytes, from 2016-10-24 21:12:27) +- ./freedreno_copyright.xml ( 1572 bytes, from 2016-10-24 21:12:27) +- ./adreno/a2xx.xml ( 32901 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_common.xml ( 12025 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_pm4.xml ( 19684 bytes, from 2016-10-24 21:12:27) +- ./adreno/a3xx.xml ( 83840 bytes, from 2016-10-24 21:12:27) +- ./adreno/a4xx.xml ( 110708 bytes, from 2016-10-24 21:12:27) +- ./adreno/a5xx.xml ( 81207 bytes, from 2016-10-26 19:36:59) +- ./adreno/ocmem.xml ( 1773 bytes, from 2016-10-24 21:12:27) -Copyright (C) 2013-2015 by the following authors: +Copyright (C) 2013-2016 by the following authors: - Rob Clark (robclark) +- Ilia Mirkin (imirkin) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -45,13 +47,18 @@ enum a4xx_color_fmt { RB4_A8_UNORM = 1, RB4_R8_UNORM = 2, + RB4_R8_SNORM = 3, + RB4_R8_UINT = 4, + RB4_R8_SINT = 5, RB4_R4G4B4A4_UNORM = 8, RB4_R5G5B5A1_UNORM = 10, - RB4_R5G6R5_UNORM = 14, + RB4_R5G6B5_UNORM = 14, RB4_R8G8_UNORM = 15, RB4_R8G8_SNORM = 16, RB4_R8G8_UINT = 17, RB4_R8G8_SINT = 18, + RB4_R16_UNORM = 19, + RB4_R16_SNORM = 20, RB4_R16_FLOAT = 21, RB4_R16_UINT = 22, RB4_R16_SINT = 23, @@ -63,12 +70,16 @@ RB4_R10G10B10A2_UNORM = 31, RB4_R10G10B10A2_UINT = 34, RB4_R11G11B10_FLOAT = 39, + RB4_R16G16_UNORM = 40, + RB4_R16G16_SNORM = 41, RB4_R16G16_FLOAT = 42, RB4_R16G16_UINT = 43, RB4_R16G16_SINT = 44, RB4_R32_FLOAT = 45, RB4_R32_UINT = 46, RB4_R32_SINT = 47, + RB4_R16G16B16A16_UNORM = 52, + RB4_R16G16B16A16_SNORM = 53, RB4_R16G16B16A16_FLOAT = 54, RB4_R16G16B16A16_UINT = 55, RB4_R16G16B16A16_SINT = 56, @@ -82,17 +93,10 @@ enum a4xx_tile_mode { TILE4_LINEAR = 0, + TILE4_2 = 2, TILE4_3 = 3, }; -enum a4xx_rb_blend_opcode { - BLEND_DST_PLUS_SRC = 0, - BLEND_SRC_MINUS_DST = 1, - BLEND_DST_MINUS_SRC = 2, - BLEND_MIN_DST_SRC = 3, - BLEND_MAX_DST_SRC = 4, -}; - enum a4xx_vtx_fmt { VFMT4_32_FLOAT = 1, VFMT4_32_32_FLOAT = 2, @@ -106,6 +110,7 @@ VFMT4_32_32_FIXED = 10, VFMT4_32_32_32_FIXED = 11, VFMT4_32_32_32_32_FIXED = 12, + VFMT4_11_11_10_FLOAT = 13, VFMT4_16_SINT = 16, VFMT4_16_16_SINT = 17, VFMT4_16_16_16_SINT = 18, @@ -146,52 +151,76 @@ VFMT4_8_8_SNORM = 53, VFMT4_8_8_8_SNORM = 54, VFMT4_8_8_8_8_SNORM = 55, - VFMT4_10_10_10_2_UINT = 60, - VFMT4_10_10_10_2_UNORM = 61, - VFMT4_10_10_10_2_SINT = 62, - VFMT4_10_10_10_2_SNORM = 63, + VFMT4_10_10_10_2_UINT = 56, + VFMT4_10_10_10_2_UNORM = 57, + VFMT4_10_10_10_2_SINT = 58, + VFMT4_10_10_10_2_SNORM = 59, + VFMT4_2_10_10_10_UINT = 60, + VFMT4_2_10_10_10_UNORM = 61, + VFMT4_2_10_10_10_SINT = 62, + VFMT4_2_10_10_10_SNORM = 63, }; enum a4xx_tex_fmt { - TFMT4_5_6_5_UNORM = 11, - TFMT4_5_5_5_1_UNORM = 10, - TFMT4_4_4_4_4_UNORM = 8, - TFMT4_X8Z24_UNORM = 71, - TFMT4_10_10_10_2_UNORM = 33, TFMT4_A8_UNORM = 3, - TFMT4_L8_A8_UNORM = 13, TFMT4_8_UNORM = 4, - TFMT4_8_8_UNORM = 14, - TFMT4_8_8_8_8_UNORM = 28, TFMT4_8_SNORM = 5, - TFMT4_8_8_SNORM = 15, - TFMT4_8_8_8_8_SNORM = 29, TFMT4_8_UINT = 6, - TFMT4_8_8_UINT = 16, - TFMT4_8_8_8_8_UINT = 30, TFMT4_8_SINT = 7, + TFMT4_4_4_4_4_UNORM = 8, + TFMT4_5_5_5_1_UNORM = 9, + TFMT4_5_6_5_UNORM = 11, + TFMT4_L8_A8_UNORM = 13, + TFMT4_8_8_UNORM = 14, + TFMT4_8_8_SNORM = 15, + TFMT4_8_8_UINT = 16, TFMT4_8_8_SINT = 17, - TFMT4_8_8_8_8_SINT = 31, + TFMT4_16_UNORM = 18, + TFMT4_16_SNORM = 19, + TFMT4_16_FLOAT = 20, TFMT4_16_UINT = 21, - TFMT4_16_16_UINT = 41, - TFMT4_16_16_16_16_UINT = 54, TFMT4_16_SINT = 22, + TFMT4_8_8_8_8_UNORM = 28, + TFMT4_8_8_8_8_SNORM = 29, + TFMT4_8_8_8_8_UINT = 30, + TFMT4_8_8_8_8_SINT = 31, + TFMT4_9_9_9_E5_FLOAT = 32, + TFMT4_10_10_10_2_UNORM = 33, + TFMT4_10_10_10_2_UINT = 34, + TFMT4_11_11_10_FLOAT = 37, + TFMT4_16_16_UNORM = 38, + TFMT4_16_16_SNORM = 39, + TFMT4_16_16_FLOAT = 40, + TFMT4_16_16_UINT = 41, TFMT4_16_16_SINT = 42, - TFMT4_16_16_16_16_SINT = 55, + TFMT4_32_FLOAT = 43, TFMT4_32_UINT = 44, - TFMT4_32_32_UINT = 57, - TFMT4_32_32_32_32_UINT = 64, TFMT4_32_SINT = 45, - TFMT4_32_32_SINT = 58, - TFMT4_32_32_32_32_SINT = 65, - TFMT4_16_FLOAT = 20, - TFMT4_16_16_FLOAT = 40, + TFMT4_16_16_16_16_UNORM = 51, + TFMT4_16_16_16_16_SNORM = 52, TFMT4_16_16_16_16_FLOAT = 53, - TFMT4_32_FLOAT = 43, + TFMT4_16_16_16_16_UINT = 54, + TFMT4_16_16_16_16_SINT = 55, TFMT4_32_32_FLOAT = 56, + TFMT4_32_32_UINT = 57, + TFMT4_32_32_SINT = 58, + TFMT4_32_32_32_FLOAT = 59, + TFMT4_32_32_32_UINT = 60, + TFMT4_32_32_32_SINT = 61, TFMT4_32_32_32_32_FLOAT = 63, - TFMT4_9_9_9_E5_FLOAT = 32, - TFMT4_11_11_10_FLOAT = 37, + TFMT4_32_32_32_32_UINT = 64, + TFMT4_32_32_32_32_SINT = 65, + TFMT4_X8Z24_UNORM = 71, + TFMT4_DXT1 = 86, + TFMT4_DXT3 = 87, + TFMT4_DXT5 = 88, + TFMT4_RGTC1_UNORM = 90, + TFMT4_RGTC1_SNORM = 91, + TFMT4_RGTC2_UNORM = 94, + TFMT4_RGTC2_SNORM = 95, + TFMT4_BPTC_UFLOAT = 97, + TFMT4_BPTC_FLOAT = 98, + TFMT4_BPTC = 99, TFMT4_ATC_RGB = 100, TFMT4_ATC_RGBA_EXPLICIT = 101, TFMT4_ATC_RGBA_INTERPOLATED = 102, @@ -240,6 +269,545 @@ EVEN_SPACING = 3, }; +enum a4xx_ccu_perfcounter_select { + CCU_BUSY_CYCLES = 0, + CCU_RB_DEPTH_RETURN_STALL = 2, + CCU_RB_COLOR_RETURN_STALL = 3, + CCU_DEPTH_BLOCKS = 6, + CCU_COLOR_BLOCKS = 7, + CCU_DEPTH_BLOCK_HIT = 8, + CCU_COLOR_BLOCK_HIT = 9, + CCU_DEPTH_FLAG1_COUNT = 10, + CCU_DEPTH_FLAG2_COUNT = 11, + CCU_DEPTH_FLAG3_COUNT = 12, + CCU_DEPTH_FLAG4_COUNT = 13, + CCU_COLOR_FLAG1_COUNT = 14, + CCU_COLOR_FLAG2_COUNT = 15, + CCU_COLOR_FLAG3_COUNT = 16, + CCU_COLOR_FLAG4_COUNT = 17, + CCU_PARTIAL_BLOCK_READ = 18, +}; + +enum a4xx_cp_perfcounter_select { + CP_ALWAYS_COUNT = 0, + CP_BUSY = 1, + CP_PFP_IDLE = 2, + CP_PFP_BUSY_WORKING = 3, + CP_PFP_STALL_CYCLES_ANY = 4, + CP_PFP_STARVE_CYCLES_ANY = 5, + CP_PFP_STARVED_PER_LOAD_ADDR = 6, + CP_PFP_STALLED_PER_STORE_ADDR = 7, + CP_PFP_PC_PROFILE = 8, + CP_PFP_MATCH_PM4_PKT_PROFILE = 9, + CP_PFP_COND_INDIRECT_DISCARDED = 10, + CP_LONG_RESUMPTIONS = 11, + CP_RESUME_CYCLES = 12, + CP_RESUME_TO_BOUNDARY_CYCLES = 13, + CP_LONG_PREEMPTIONS = 14, + CP_PREEMPT_CYCLES = 15, + CP_PREEMPT_TO_BOUNDARY_CYCLES = 16, + CP_ME_FIFO_EMPTY_PFP_IDLE = 17, + CP_ME_FIFO_EMPTY_PFP_BUSY = 18, + CP_ME_FIFO_NOT_EMPTY_NOT_FULL = 19, + CP_ME_FIFO_FULL_ME_BUSY = 20, + CP_ME_FIFO_FULL_ME_NON_WORKING = 21, + CP_ME_WAITING_FOR_PACKETS = 22, + CP_ME_BUSY_WORKING = 23, + CP_ME_STARVE_CYCLES_ANY = 24, + CP_ME_STARVE_CYCLES_PER_PROFILE = 25, + CP_ME_STALL_CYCLES_PER_PROFILE = 26, + CP_ME_PC_PROFILE = 27, + CP_RCIU_FIFO_EMPTY = 28, + CP_RCIU_FIFO_NOT_EMPTY_NOT_FULL = 29, + CP_RCIU_FIFO_FULL = 30, + CP_RCIU_FIFO_FULL_NO_CONTEXT = 31, + CP_RCIU_FIFO_FULL_AHB_MASTER = 32, + CP_RCIU_FIFO_FULL_OTHER = 33, + CP_AHB_IDLE = 34, + CP_AHB_STALL_ON_GRANT_NO_SPLIT = 35, + CP_AHB_STALL_ON_GRANT_SPLIT = 36, + CP_AHB_STALL_ON_GRANT_SPLIT_PROFILE = 37, + CP_AHB_BUSY_WORKING = 38, + CP_AHB_BUSY_STALL_ON_HRDY = 39, + CP_AHB_BUSY_STALL_ON_HRDY_PROFILE = 40, +}; + +enum a4xx_gras_ras_perfcounter_select { + RAS_SUPER_TILES = 0, + RAS_8X8_TILES = 1, + RAS_4X4_TILES = 2, + RAS_BUSY_CYCLES = 3, + RAS_STALL_CYCLES_BY_RB = 4, + RAS_STALL_CYCLES_BY_VSC = 5, + RAS_STARVE_CYCLES_BY_TSE = 6, + RAS_SUPERTILE_CYCLES = 7, + RAS_TILE_CYCLES = 8, + RAS_FULLY_COVERED_SUPER_TILES = 9, + RAS_FULLY_COVERED_8X8_TILES = 10, + RAS_4X4_PRIM = 11, + RAS_8X4_4X8_PRIM = 12, + RAS_8X8_PRIM = 13, +}; + +enum a4xx_gras_tse_perfcounter_select { + TSE_INPUT_PRIM = 0, + TSE_INPUT_NULL_PRIM = 1, + TSE_TRIVAL_REJ_PRIM = 2, + TSE_CLIPPED_PRIM = 3, + TSE_NEW_PRIM = 4, + TSE_ZERO_AREA_PRIM = 5, + TSE_FACENESS_CULLED_PRIM = 6, + TSE_ZERO_PIXEL_PRIM = 7, + TSE_OUTPUT_NULL_PRIM = 8, + TSE_OUTPUT_VISIBLE_PRIM = 9, + TSE_PRE_CLIP_PRIM = 10, + TSE_POST_CLIP_PRIM = 11, + TSE_BUSY_CYCLES = 12, + TSE_PC_STARVE = 13, + TSE_RAS_STALL = 14, + TSE_STALL_BARYPLANE_FIFO_FULL = 15, + TSE_STALL_ZPLANE_FIFO_FULL = 16, +}; + +enum a4xx_hlsq_perfcounter_select { + HLSQ_SP_VS_STAGE_CONSTANT = 0, + HLSQ_SP_VS_STAGE_INSTRUCTIONS = 1, + HLSQ_SP_FS_STAGE_CONSTANT = 2, + HLSQ_SP_FS_STAGE_INSTRUCTIONS = 3, + HLSQ_TP_STATE = 4, + HLSQ_QUADS = 5, + HLSQ_PIXELS = 6, + HLSQ_VERTICES = 7, + HLSQ_SP_VS_STAGE_DATA_BYTES = 13, + HLSQ_SP_FS_STAGE_DATA_BYTES = 14, + HLSQ_BUSY_CYCLES = 15, + HLSQ_STALL_CYCLES_SP_STATE = 16, + HLSQ_STALL_CYCLES_SP_VS_STAGE = 17, + HLSQ_STALL_CYCLES_SP_FS_STAGE = 18, + HLSQ_STALL_CYCLES_UCHE = 19, + HLSQ_RBBM_LOAD_CYCLES = 20, + HLSQ_DI_TO_VS_START_SP = 21, + HLSQ_DI_TO_FS_START_SP = 22, + HLSQ_VS_STAGE_START_TO_DONE_SP = 23, + HLSQ_FS_STAGE_START_TO_DONE_SP = 24, + HLSQ_SP_STATE_COPY_CYCLES_VS_STAGE = 25, + HLSQ_SP_STATE_COPY_CYCLES_FS_STAGE = 26, + HLSQ_UCHE_LATENCY_CYCLES = 27, + HLSQ_UCHE_LATENCY_COUNT = 28, + HLSQ_STARVE_CYCLES_VFD = 29, +}; + +enum a4xx_pc_perfcounter_select { + PC_VIS_STREAMS_LOADED = 0, + PC_VPC_PRIMITIVES = 2, + PC_DEAD_PRIM = 3, + PC_LIVE_PRIM = 4, + PC_DEAD_DRAWCALLS = 5, + PC_LIVE_DRAWCALLS = 6, + PC_VERTEX_MISSES = 7, + PC_STALL_CYCLES_VFD = 9, + PC_STALL_CYCLES_TSE = 10, + PC_STALL_CYCLES_UCHE = 11, + PC_WORKING_CYCLES = 12, + PC_IA_VERTICES = 13, + PC_GS_PRIMITIVES = 14, + PC_HS_INVOCATIONS = 15, + PC_DS_INVOCATIONS = 16, + PC_DS_PRIMITIVES = 17, + PC_STARVE_CYCLES_FOR_INDEX = 20, + PC_STARVE_CYCLES_FOR_TESS_FACTOR = 21, + PC_STARVE_CYCLES_FOR_VIZ_STREAM = 22, + PC_STALL_CYCLES_TESS = 23, + PC_STARVE_CYCLES_FOR_POSITION = 24, + PC_MODE0_DRAWCALL = 25, + PC_MODE1_DRAWCALL = 26, + PC_MODE2_DRAWCALL = 27, + PC_MODE3_DRAWCALL = 28, + PC_MODE4_DRAWCALL = 29, + PC_PREDICATED_DEAD_DRAWCALL = 30, + PC_STALL_CYCLES_BY_TSE_ONLY = 31, + PC_STALL_CYCLES_BY_VPC_ONLY = 32, + PC_VPC_POS_DATA_TRANSACTION = 33, + PC_BUSY_CYCLES = 34, + PC_STARVE_CYCLES_DI = 35, + PC_STALL_CYCLES_VPC = 36, + TESS_WORKING_CYCLES = 37, + TESS_NUM_CYCLES_SETUP_WORKING = 38, + TESS_NUM_CYCLES_PTGEN_WORKING = 39, + TESS_NUM_CYCLES_CONNGEN_WORKING = 40, + TESS_BUSY_CYCLES = 41, + TESS_STARVE_CYCLES_PC = 42, + TESS_STALL_CYCLES_PC = 43, +}; + +enum a4xx_pwr_perfcounter_select { + PWR_CORE_CLOCK_CYCLES = 0, + PWR_BUSY_CLOCK_CYCLES = 1, +}; + +enum a4xx_rb_perfcounter_select { + RB_BUSY_CYCLES = 0, + RB_BUSY_CYCLES_BINNING = 1, + RB_BUSY_CYCLES_RENDERING = 2, + RB_BUSY_CYCLES_RESOLVE = 3, + RB_STARVE_CYCLES_BY_SP = 4, + RB_STARVE_CYCLES_BY_RAS = 5, + RB_STARVE_CYCLES_BY_MARB = 6, + RB_STALL_CYCLES_BY_MARB = 7, + RB_STALL_CYCLES_BY_HLSQ = 8, + RB_RB_RB_MARB_DATA = 9, + RB_SP_RB_QUAD = 10, + RB_RAS_RB_Z_QUADS = 11, + RB_GMEM_CH0_READ = 12, + RB_GMEM_CH1_READ = 13, + RB_GMEM_CH0_WRITE = 14, + RB_GMEM_CH1_WRITE = 15, + RB_CP_CONTEXT_DONE = 16, + RB_CP_CACHE_FLUSH = 17, + RB_CP_ZPASS_DONE = 18, + RB_STALL_FIFO0_FULL = 19, + RB_STALL_FIFO1_FULL = 20, + RB_STALL_FIFO2_FULL = 21, + RB_STALL_FIFO3_FULL = 22, + RB_RB_HLSQ_TRANSACTIONS = 23, + RB_Z_READ = 24, + RB_Z_WRITE = 25, + RB_C_READ = 26, + RB_C_WRITE = 27, + RB_C_READ_LATENCY = 28, + RB_Z_READ_LATENCY = 29, + RB_STALL_BY_UCHE = 30, + RB_MARB_UCHE_TRANSACTIONS = 31, + RB_CACHE_STALL_MISS = 32, + RB_CACHE_STALL_FIFO_FULL = 33, + RB_8BIT_BLENDER_UNITS_ACTIVE = 34, + RB_16BIT_BLENDER_UNITS_ACTIVE = 35, + RB_SAMPLER_UNITS_ACTIVE = 36, + RB_TOTAL_PASS = 38, + RB_Z_PASS = 39, + RB_Z_FAIL = 40, + RB_S_FAIL = 41, + RB_POWER0 = 42, + RB_POWER1 = 43, + RB_POWER2 = 44, + RB_POWER3 = 45, + RB_POWER4 = 46, + RB_POWER5 = 47, + RB_POWER6 = 48, + RB_POWER7 = 49, +}; + +enum a4xx_rbbm_perfcounter_select { + RBBM_ALWAYS_ON = 0, + RBBM_VBIF_BUSY = 1, + RBBM_TSE_BUSY = 2, + RBBM_RAS_BUSY = 3, + RBBM_PC_DCALL_BUSY = 4, + RBBM_PC_VSD_BUSY = 5, + RBBM_VFD_BUSY = 6, + RBBM_VPC_BUSY = 7, + RBBM_UCHE_BUSY = 8, + RBBM_VSC_BUSY = 9, + RBBM_HLSQ_BUSY = 10, + RBBM_ANY_RB_BUSY = 11, + RBBM_ANY_TPL1_BUSY = 12, + RBBM_ANY_SP_BUSY = 13, + RBBM_ANY_MARB_BUSY = 14, + RBBM_ANY_ARB_BUSY = 15, + RBBM_AHB_STATUS_BUSY = 16, + RBBM_AHB_STATUS_STALLED = 17, + RBBM_AHB_STATUS_TXFR = 18, + RBBM_AHB_STATUS_TXFR_SPLIT = 19, + RBBM_AHB_STATUS_TXFR_ERROR = 20, + RBBM_AHB_STATUS_LONG_STALL = 21, + RBBM_STATUS_MASKED = 22, + RBBM_CP_BUSY_GFX_CORE_IDLE = 23, + RBBM_TESS_BUSY = 24, + RBBM_COM_BUSY = 25, + RBBM_DCOM_BUSY = 32, + RBBM_ANY_CCU_BUSY = 33, + RBBM_DPM_BUSY = 34, +}; + +enum a4xx_sp_perfcounter_select { + SP_LM_LOAD_INSTRUCTIONS = 0, + SP_LM_STORE_INSTRUCTIONS = 1, + SP_LM_ATOMICS = 2, + SP_GM_LOAD_INSTRUCTIONS = 3, + SP_GM_STORE_INSTRUCTIONS = 4, + SP_GM_ATOMICS = 5, + SP_VS_STAGE_TEX_INSTRUCTIONS = 6, + SP_VS_STAGE_CFLOW_INSTRUCTIONS = 7, + SP_VS_STAGE_EFU_INSTRUCTIONS = 8, + SP_VS_STAGE_FULL_ALU_INSTRUCTIONS = 9, + SP_VS_STAGE_HALF_ALU_INSTRUCTIONS = 10, + SP_FS_STAGE_TEX_INSTRUCTIONS = 11, + SP_FS_STAGE_CFLOW_INSTRUCTIONS = 12, + SP_FS_STAGE_EFU_INSTRUCTIONS = 13, + SP_FS_STAGE_FULL_ALU_INSTRUCTIONS = 14, + SP_FS_STAGE_HALF_ALU_INSTRUCTIONS = 15, + SP_VS_INSTRUCTIONS = 17, + SP_FS_INSTRUCTIONS = 18, + SP_ADDR_LOCK_COUNT = 19, + SP_UCHE_READ_TRANS = 20, + SP_UCHE_WRITE_TRANS = 21, + SP_EXPORT_VPC_TRANS = 22, + SP_EXPORT_RB_TRANS = 23, + SP_PIXELS_KILLED = 24, + SP_ICL1_REQUESTS = 25, + SP_ICL1_MISSES = 26, + SP_ICL0_REQUESTS = 27, + SP_ICL0_MISSES = 28, + SP_ALU_WORKING_CYCLES = 29, + SP_EFU_WORKING_CYCLES = 30, + SP_STALL_CYCLES_BY_VPC = 31, + SP_STALL_CYCLES_BY_TP = 32, + SP_STALL_CYCLES_BY_UCHE = 33, + SP_STALL_CYCLES_BY_RB = 34, + SP_BUSY_CYCLES = 35, + SP_HS_INSTRUCTIONS = 36, + SP_DS_INSTRUCTIONS = 37, + SP_GS_INSTRUCTIONS = 38, + SP_CS_INSTRUCTIONS = 39, + SP_SCHEDULER_NON_WORKING = 40, + SP_WAVE_CONTEXTS = 41, + SP_WAVE_CONTEXT_CYCLES = 42, + SP_POWER0 = 43, + SP_POWER1 = 44, + SP_POWER2 = 45, + SP_POWER3 = 46, + SP_POWER4 = 47, + SP_POWER5 = 48, + SP_POWER6 = 49, + SP_POWER7 = 50, + SP_POWER8 = 51, + SP_POWER9 = 52, + SP_POWER10 = 53, + SP_POWER11 = 54, + SP_POWER12 = 55, + SP_POWER13 = 56, + SP_POWER14 = 57, + SP_POWER15 = 58, +}; + +enum a4xx_tp_perfcounter_select { + TP_L1_REQUESTS = 0, + TP_L1_MISSES = 1, + TP_QUADS_OFFSET = 8, + TP_QUAD_SHADOW = 9, + TP_QUADS_ARRAY = 10, + TP_QUADS_GRADIENT = 11, + TP_QUADS_1D2D = 12, + TP_QUADS_3DCUBE = 13, + TP_BUSY_CYCLES = 16, + TP_STALL_CYCLES_BY_ARB = 17, + TP_STATE_CACHE_REQUESTS = 20, + TP_STATE_CACHE_MISSES = 21, + TP_POWER0 = 22, + TP_POWER1 = 23, + TP_POWER2 = 24, + TP_POWER3 = 25, + TP_POWER4 = 26, + TP_POWER5 = 27, + TP_POWER6 = 28, + TP_POWER7 = 29, +}; + +enum a4xx_uche_perfcounter_select { + UCHE_VBIF_READ_BEATS_TP = 0, + UCHE_VBIF_READ_BEATS_VFD = 1, + UCHE_VBIF_READ_BEATS_HLSQ = 2, + UCHE_VBIF_READ_BEATS_MARB = 3, + UCHE_VBIF_READ_BEATS_SP = 4, + UCHE_READ_REQUESTS_TP = 5, + UCHE_READ_REQUESTS_VFD = 6, + UCHE_READ_REQUESTS_HLSQ = 7, + UCHE_READ_REQUESTS_MARB = 8, + UCHE_READ_REQUESTS_SP = 9, + UCHE_WRITE_REQUESTS_MARB = 10, + UCHE_WRITE_REQUESTS_SP = 11, + UCHE_TAG_CHECK_FAILS = 12, + UCHE_EVICTS = 13, + UCHE_FLUSHES = 14, + UCHE_VBIF_LATENCY_CYCLES = 15, + UCHE_VBIF_LATENCY_SAMPLES = 16, + UCHE_BUSY_CYCLES = 17, + UCHE_VBIF_READ_BEATS_PC = 18, + UCHE_READ_REQUESTS_PC = 19, + UCHE_WRITE_REQUESTS_VPC = 20, + UCHE_STALL_BY_VBIF = 21, + UCHE_WRITE_REQUESTS_VSC = 22, + UCHE_POWER0 = 23, + UCHE_POWER1 = 24, + UCHE_POWER2 = 25, + UCHE_POWER3 = 26, + UCHE_POWER4 = 27, + UCHE_POWER5 = 28, + UCHE_POWER6 = 29, + UCHE_POWER7 = 30, +}; + +enum a4xx_vbif_perfcounter_select { + AXI_READ_REQUESTS_ID_0 = 0, + AXI_READ_REQUESTS_ID_1 = 1, + AXI_READ_REQUESTS_ID_2 = 2, + AXI_READ_REQUESTS_ID_3 = 3, + AXI_READ_REQUESTS_ID_4 = 4, + AXI_READ_REQUESTS_ID_5 = 5, + AXI_READ_REQUESTS_ID_6 = 6, + AXI_READ_REQUESTS_ID_7 = 7, + AXI_READ_REQUESTS_ID_8 = 8, + AXI_READ_REQUESTS_ID_9 = 9, + AXI_READ_REQUESTS_ID_10 = 10, + AXI_READ_REQUESTS_ID_11 = 11, + AXI_READ_REQUESTS_ID_12 = 12, + AXI_READ_REQUESTS_ID_13 = 13, + AXI_READ_REQUESTS_ID_14 = 14, + AXI_READ_REQUESTS_ID_15 = 15, + AXI0_READ_REQUESTS_TOTAL = 16, + AXI1_READ_REQUESTS_TOTAL = 17, + AXI2_READ_REQUESTS_TOTAL = 18, + AXI3_READ_REQUESTS_TOTAL = 19, + AXI_READ_REQUESTS_TOTAL = 20, + AXI_WRITE_REQUESTS_ID_0 = 21, + AXI_WRITE_REQUESTS_ID_1 = 22, + AXI_WRITE_REQUESTS_ID_2 = 23, + AXI_WRITE_REQUESTS_ID_3 = 24, + AXI_WRITE_REQUESTS_ID_4 = 25, + AXI_WRITE_REQUESTS_ID_5 = 26, + AXI_WRITE_REQUESTS_ID_6 = 27, + AXI_WRITE_REQUESTS_ID_7 = 28, + AXI_WRITE_REQUESTS_ID_8 = 29, + AXI_WRITE_REQUESTS_ID_9 = 30, + AXI_WRITE_REQUESTS_ID_10 = 31, + AXI_WRITE_REQUESTS_ID_11 = 32, + AXI_WRITE_REQUESTS_ID_12 = 33, + AXI_WRITE_REQUESTS_ID_13 = 34, + AXI_WRITE_REQUESTS_ID_14 = 35, + AXI_WRITE_REQUESTS_ID_15 = 36, + AXI0_WRITE_REQUESTS_TOTAL = 37, + AXI1_WRITE_REQUESTS_TOTAL = 38, + AXI2_WRITE_REQUESTS_TOTAL = 39, + AXI3_WRITE_REQUESTS_TOTAL = 40, + AXI_WRITE_REQUESTS_TOTAL = 41, + AXI_TOTAL_REQUESTS = 42, + AXI_READ_DATA_BEATS_ID_0 = 43, + AXI_READ_DATA_BEATS_ID_1 = 44, + AXI_READ_DATA_BEATS_ID_2 = 45, + AXI_READ_DATA_BEATS_ID_3 = 46, + AXI_READ_DATA_BEATS_ID_4 = 47, + AXI_READ_DATA_BEATS_ID_5 = 48, + AXI_READ_DATA_BEATS_ID_6 = 49, + AXI_READ_DATA_BEATS_ID_7 = 50, + AXI_READ_DATA_BEATS_ID_8 = 51, + AXI_READ_DATA_BEATS_ID_9 = 52, + AXI_READ_DATA_BEATS_ID_10 = 53, + AXI_READ_DATA_BEATS_ID_11 = 54, + AXI_READ_DATA_BEATS_ID_12 = 55, + AXI_READ_DATA_BEATS_ID_13 = 56, + AXI_READ_DATA_BEATS_ID_14 = 57, + AXI_READ_DATA_BEATS_ID_15 = 58, + AXI0_READ_DATA_BEATS_TOTAL = 59, + AXI1_READ_DATA_BEATS_TOTAL = 60, + AXI2_READ_DATA_BEATS_TOTAL = 61, + AXI3_READ_DATA_BEATS_TOTAL = 62, + AXI_READ_DATA_BEATS_TOTAL = 63, + AXI_WRITE_DATA_BEATS_ID_0 = 64, + AXI_WRITE_DATA_BEATS_ID_1 = 65, + AXI_WRITE_DATA_BEATS_ID_2 = 66, + AXI_WRITE_DATA_BEATS_ID_3 = 67, + AXI_WRITE_DATA_BEATS_ID_4 = 68, + AXI_WRITE_DATA_BEATS_ID_5 = 69, + AXI_WRITE_DATA_BEATS_ID_6 = 70, + AXI_WRITE_DATA_BEATS_ID_7 = 71, + AXI_WRITE_DATA_BEATS_ID_8 = 72, + AXI_WRITE_DATA_BEATS_ID_9 = 73, + AXI_WRITE_DATA_BEATS_ID_10 = 74, + AXI_WRITE_DATA_BEATS_ID_11 = 75, + AXI_WRITE_DATA_BEATS_ID_12 = 76, + AXI_WRITE_DATA_BEATS_ID_13 = 77, + AXI_WRITE_DATA_BEATS_ID_14 = 78, + AXI_WRITE_DATA_BEATS_ID_15 = 79, + AXI0_WRITE_DATA_BEATS_TOTAL = 80, + AXI1_WRITE_DATA_BEATS_TOTAL = 81, + AXI2_WRITE_DATA_BEATS_TOTAL = 82, + AXI3_WRITE_DATA_BEATS_TOTAL = 83, + AXI_WRITE_DATA_BEATS_TOTAL = 84, + AXI_DATA_BEATS_TOTAL = 85, + CYCLES_HELD_OFF_ID_0 = 86, + CYCLES_HELD_OFF_ID_1 = 87, + CYCLES_HELD_OFF_ID_2 = 88, + CYCLES_HELD_OFF_ID_3 = 89, + CYCLES_HELD_OFF_ID_4 = 90, + CYCLES_HELD_OFF_ID_5 = 91, + CYCLES_HELD_OFF_ID_6 = 92, + CYCLES_HELD_OFF_ID_7 = 93, + CYCLES_HELD_OFF_ID_8 = 94, + CYCLES_HELD_OFF_ID_9 = 95, + CYCLES_HELD_OFF_ID_10 = 96, + CYCLES_HELD_OFF_ID_11 = 97, + CYCLES_HELD_OFF_ID_12 = 98, + CYCLES_HELD_OFF_ID_13 = 99, + CYCLES_HELD_OFF_ID_14 = 100, + CYCLES_HELD_OFF_ID_15 = 101, + AXI_READ_REQUEST_HELD_OFF = 102, + AXI_WRITE_REQUEST_HELD_OFF = 103, + AXI_REQUEST_HELD_OFF = 104, + AXI_WRITE_DATA_HELD_OFF = 105, + OCMEM_AXI_READ_REQUEST_HELD_OFF = 106, + OCMEM_AXI_WRITE_REQUEST_HELD_OFF = 107, + OCMEM_AXI_REQUEST_HELD_OFF = 108, + OCMEM_AXI_WRITE_DATA_HELD_OFF = 109, + ELAPSED_CYCLES_DDR = 110, + ELAPSED_CYCLES_OCMEM = 111, +}; + +enum a4xx_vfd_perfcounter_select { + VFD_UCHE_BYTE_FETCHED = 0, + VFD_UCHE_TRANS = 1, + VFD_FETCH_INSTRUCTIONS = 3, + VFD_BUSY_CYCLES = 5, + VFD_STALL_CYCLES_UCHE = 6, + VFD_STALL_CYCLES_HLSQ = 7, + VFD_STALL_CYCLES_VPC_BYPASS = 8, + VFD_STALL_CYCLES_VPC_ALLOC = 9, + VFD_MODE_0_FIBERS = 13, + VFD_MODE_1_FIBERS = 14, + VFD_MODE_2_FIBERS = 15, + VFD_MODE_3_FIBERS = 16, + VFD_MODE_4_FIBERS = 17, + VFD_BFIFO_STALL = 18, + VFD_NUM_VERTICES_TOTAL = 19, + VFD_PACKER_FULL = 20, + VFD_UCHE_REQUEST_FIFO_FULL = 21, + VFD_STARVE_CYCLES_PC = 22, + VFD_STARVE_CYCLES_UCHE = 23, +}; + +enum a4xx_vpc_perfcounter_select { + VPC_SP_LM_COMPONENTS = 2, + VPC_SP0_LM_BYTES = 3, + VPC_SP1_LM_BYTES = 4, + VPC_SP2_LM_BYTES = 5, + VPC_SP3_LM_BYTES = 6, + VPC_WORKING_CYCLES = 7, + VPC_STALL_CYCLES_LM = 8, + VPC_STARVE_CYCLES_RAS = 9, + VPC_STREAMOUT_CYCLES = 10, + VPC_UCHE_TRANSACTIONS = 12, + VPC_STALL_CYCLES_UCHE = 13, + VPC_BUSY_CYCLES = 14, + VPC_STARVE_CYCLES_SP = 15, +}; + +enum a4xx_vsc_perfcounter_select { + VSC_BUSY_CYCLES = 0, + VSC_WORKING_CYCLES = 1, + VSC_STALL_CYCLES_UCHE = 2, + VSC_STARVE_CYCLES_RAS = 3, + VSC_EOT_NUM = 4, +}; + enum a4xx_tex_filter { A4XX_TEX_NEAREST = 0, A4XX_TEX_LINEAR = 1, @@ -326,6 +894,12 @@ #define REG_A4XX_RB_PERFCTR_RB_SEL_7 0x00000cce +#define REG_A4XX_RB_PERFCTR_CCU_SEL_0 0x00000ccf + +#define REG_A4XX_RB_PERFCTR_CCU_SEL_1 0x00000cd0 + +#define REG_A4XX_RB_PERFCTR_CCU_SEL_2 0x00000cd1 + #define REG_A4XX_RB_PERFCTR_CCU_SEL_3 0x00000cd2 #define REG_A4XX_RB_FRAME_BUFFER_DIMENSION 0x00000ce0 @@ -363,6 +937,7 @@ { return ((val >> 5) << A4XX_RB_MODE_CONTROL_HEIGHT__SHIFT) & A4XX_RB_MODE_CONTROL_HEIGHT__MASK; } +#define A4XX_RB_MODE_CONTROL_ENABLE_GMEM 0x00010000 #define REG_A4XX_RB_RENDER_CONTROL 0x000020a1 #define A4XX_RB_RENDER_CONTROL_BINNING_PASS 0x00000001 @@ -400,8 +975,13 @@ #define A4XX_RB_MRT_CONTROL_READ_DEST_ENABLE 0x00000008 #define A4XX_RB_MRT_CONTROL_BLEND 0x00000010 #define A4XX_RB_MRT_CONTROL_BLEND2 0x00000020 -#define A4XX_RB_MRT_CONTROL_FASTCLEAR 0x00000400 -#define A4XX_RB_MRT_CONTROL_B11 0x00000800 +#define A4XX_RB_MRT_CONTROL_ROP_ENABLE 0x00000040 +#define A4XX_RB_MRT_CONTROL_ROP_CODE__MASK 0x00000f00 +#define A4XX_RB_MRT_CONTROL_ROP_CODE__SHIFT 8 +static inline uint32_t A4XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val) +{ + return ((val) << A4XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A4XX_RB_MRT_CONTROL_ROP_CODE__MASK; +} #define A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK 0x0f000000 #define A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT 24 static inline uint32_t A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val) @@ -461,7 +1041,7 @@ } #define A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK 0x000000e0 #define A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT 5 -static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a4xx_rb_blend_opcode val) +static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val) { return ((val) << A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK; } @@ -479,7 +1059,7 @@ } #define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK 0x00e00000 #define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT 21 -static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a4xx_rb_blend_opcode val) +static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val) { return ((val) << A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK; } @@ -490,13 +1070,19 @@ return ((val) << A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK; } -#define REG_A4XX_RB_BLEND_RED 0x000020f3 -#define A4XX_RB_BLEND_RED_UINT__MASK 0x00007fff +#define REG_A4XX_RB_BLEND_RED 0x000020f0 +#define A4XX_RB_BLEND_RED_UINT__MASK 0x000000ff #define A4XX_RB_BLEND_RED_UINT__SHIFT 0 static inline uint32_t A4XX_RB_BLEND_RED_UINT(uint32_t val) { return ((val) << A4XX_RB_BLEND_RED_UINT__SHIFT) & A4XX_RB_BLEND_RED_UINT__MASK; } +#define A4XX_RB_BLEND_RED_SINT__MASK 0x0000ff00 +#define A4XX_RB_BLEND_RED_SINT__SHIFT 8 +static inline uint32_t A4XX_RB_BLEND_RED_SINT(uint32_t val) +{ + return ((val) << A4XX_RB_BLEND_RED_SINT__SHIFT) & A4XX_RB_BLEND_RED_SINT__MASK; +} #define A4XX_RB_BLEND_RED_FLOAT__MASK 0xffff0000 #define A4XX_RB_BLEND_RED_FLOAT__SHIFT 16 static inline uint32_t A4XX_RB_BLEND_RED_FLOAT(float val) @@ -504,13 +1090,27 @@ return ((util_float_to_half(val)) << A4XX_RB_BLEND_RED_FLOAT__SHIFT) & A4XX_RB_BLEND_RED_FLOAT__MASK; } -#define REG_A4XX_RB_BLEND_GREEN 0x000020f4 -#define A4XX_RB_BLEND_GREEN_UINT__MASK 0x00007fff +#define REG_A4XX_RB_BLEND_RED_F32 0x000020f1 +#define A4XX_RB_BLEND_RED_F32__MASK 0xffffffff +#define A4XX_RB_BLEND_RED_F32__SHIFT 0 +static inline uint32_t A4XX_RB_BLEND_RED_F32(float val) +{ + return ((fui(val)) << A4XX_RB_BLEND_RED_F32__SHIFT) & A4XX_RB_BLEND_RED_F32__MASK; +} + +#define REG_A4XX_RB_BLEND_GREEN 0x000020f2 +#define A4XX_RB_BLEND_GREEN_UINT__MASK 0x000000ff #define A4XX_RB_BLEND_GREEN_UINT__SHIFT 0 static inline uint32_t A4XX_RB_BLEND_GREEN_UINT(uint32_t val) { return ((val) << A4XX_RB_BLEND_GREEN_UINT__SHIFT) & A4XX_RB_BLEND_GREEN_UINT__MASK; } +#define A4XX_RB_BLEND_GREEN_SINT__MASK 0x0000ff00 +#define A4XX_RB_BLEND_GREEN_SINT__SHIFT 8 +static inline uint32_t A4XX_RB_BLEND_GREEN_SINT(uint32_t val) +{ + return ((val) << A4XX_RB_BLEND_GREEN_SINT__SHIFT) & A4XX_RB_BLEND_GREEN_SINT__MASK; +} #define A4XX_RB_BLEND_GREEN_FLOAT__MASK 0xffff0000 #define A4XX_RB_BLEND_GREEN_FLOAT__SHIFT 16 static inline uint32_t A4XX_RB_BLEND_GREEN_FLOAT(float val) @@ -518,13 +1118,27 @@ return ((util_float_to_half(val)) << A4XX_RB_BLEND_GREEN_FLOAT__SHIFT) & A4XX_RB_BLEND_GREEN_FLOAT__MASK; } -#define REG_A4XX_RB_BLEND_BLUE 0x000020f5 -#define A4XX_RB_BLEND_BLUE_UINT__MASK 0x00007fff +#define REG_A4XX_RB_BLEND_GREEN_F32 0x000020f3 +#define A4XX_RB_BLEND_GREEN_F32__MASK 0xffffffff +#define A4XX_RB_BLEND_GREEN_F32__SHIFT 0 +static inline uint32_t A4XX_RB_BLEND_GREEN_F32(float val) +{ + return ((fui(val)) << A4XX_RB_BLEND_GREEN_F32__SHIFT) & A4XX_RB_BLEND_GREEN_F32__MASK; +} + +#define REG_A4XX_RB_BLEND_BLUE 0x000020f4 +#define A4XX_RB_BLEND_BLUE_UINT__MASK 0x000000ff #define A4XX_RB_BLEND_BLUE_UINT__SHIFT 0 static inline uint32_t A4XX_RB_BLEND_BLUE_UINT(uint32_t val) { return ((val) << A4XX_RB_BLEND_BLUE_UINT__SHIFT) & A4XX_RB_BLEND_BLUE_UINT__MASK; } +#define A4XX_RB_BLEND_BLUE_SINT__MASK 0x0000ff00 +#define A4XX_RB_BLEND_BLUE_SINT__SHIFT 8 +static inline uint32_t A4XX_RB_BLEND_BLUE_SINT(uint32_t val) +{ + return ((val) << A4XX_RB_BLEND_BLUE_SINT__SHIFT) & A4XX_RB_BLEND_BLUE_SINT__MASK; +} #define A4XX_RB_BLEND_BLUE_FLOAT__MASK 0xffff0000 #define A4XX_RB_BLEND_BLUE_FLOAT__SHIFT 16 static inline uint32_t A4XX_RB_BLEND_BLUE_FLOAT(float val) @@ -532,13 +1146,27 @@ return ((util_float_to_half(val)) << A4XX_RB_BLEND_BLUE_FLOAT__SHIFT) & A4XX_RB_BLEND_BLUE_FLOAT__MASK; } +#define REG_A4XX_RB_BLEND_BLUE_F32 0x000020f5 +#define A4XX_RB_BLEND_BLUE_F32__MASK 0xffffffff +#define A4XX_RB_BLEND_BLUE_F32__SHIFT 0 +static inline uint32_t A4XX_RB_BLEND_BLUE_F32(float val) +{ + return ((fui(val)) << A4XX_RB_BLEND_BLUE_F32__SHIFT) & A4XX_RB_BLEND_BLUE_F32__MASK; +} + #define REG_A4XX_RB_BLEND_ALPHA 0x000020f6 -#define A4XX_RB_BLEND_ALPHA_UINT__MASK 0x00007fff +#define A4XX_RB_BLEND_ALPHA_UINT__MASK 0x000000ff #define A4XX_RB_BLEND_ALPHA_UINT__SHIFT 0 static inline uint32_t A4XX_RB_BLEND_ALPHA_UINT(uint32_t val) { return ((val) << A4XX_RB_BLEND_ALPHA_UINT__SHIFT) & A4XX_RB_BLEND_ALPHA_UINT__MASK; } +#define A4XX_RB_BLEND_ALPHA_SINT__MASK 0x0000ff00 +#define A4XX_RB_BLEND_ALPHA_SINT__SHIFT 8 +static inline uint32_t A4XX_RB_BLEND_ALPHA_SINT(uint32_t val) +{ + return ((val) << A4XX_RB_BLEND_ALPHA_SINT__SHIFT) & A4XX_RB_BLEND_ALPHA_SINT__MASK; +} #define A4XX_RB_BLEND_ALPHA_FLOAT__MASK 0xffff0000 #define A4XX_RB_BLEND_ALPHA_FLOAT__SHIFT 16 static inline uint32_t A4XX_RB_BLEND_ALPHA_FLOAT(float val) @@ -546,6 +1174,14 @@ return ((util_float_to_half(val)) << A4XX_RB_BLEND_ALPHA_FLOAT__SHIFT) & A4XX_RB_BLEND_ALPHA_FLOAT__MASK; } +#define REG_A4XX_RB_BLEND_ALPHA_F32 0x000020f7 +#define A4XX_RB_BLEND_ALPHA_F32__MASK 0xffffffff +#define A4XX_RB_BLEND_ALPHA_F32__SHIFT 0 +static inline uint32_t A4XX_RB_BLEND_ALPHA_F32(float val) +{ + return ((fui(val)) << A4XX_RB_BLEND_ALPHA_F32__SHIFT) & A4XX_RB_BLEND_ALPHA_F32__MASK; +} + #define REG_A4XX_RB_ALPHA_CONTROL 0x000020f8 #define A4XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK 0x000000ff #define A4XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT 0 @@ -568,7 +1204,7 @@ { return ((val) << A4XX_RB_FS_OUTPUT_ENABLE_BLEND__SHIFT) & A4XX_RB_FS_OUTPUT_ENABLE_BLEND__MASK; } -#define A4XX_RB_FS_OUTPUT_FAST_CLEAR 0x00000100 +#define A4XX_RB_FS_OUTPUT_INDEPENDENT_BLEND 0x00000100 #define A4XX_RB_FS_OUTPUT_SAMPLE_MASK__MASK 0xffff0000 #define A4XX_RB_FS_OUTPUT_SAMPLE_MASK__SHIFT 16 static inline uint32_t A4XX_RB_FS_OUTPUT_SAMPLE_MASK(uint32_t val) @@ -734,8 +1370,9 @@ { return ((val) << A4XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT) & A4XX_RB_DEPTH_CONTROL_ZFUNC__MASK; } -#define A4XX_RB_DEPTH_CONTROL_BF_ENABLE 0x00000080 +#define A4XX_RB_DEPTH_CONTROL_Z_CLAMP_ENABLE 0x00000080 #define A4XX_RB_DEPTH_CONTROL_EARLY_Z_DISABLE 0x00010000 +#define A4XX_RB_DEPTH_CONTROL_FORCE_FRAGZ_TO_FS 0x00020000 #define A4XX_RB_DEPTH_CONTROL_Z_TEST_ENABLE 0x80000000 #define REG_A4XX_RB_DEPTH_CLEAR 0x00002102 @@ -996,8 +1633,386 @@ #define REG_A4XX_RBBM_CFG_DEBBUS_SEL_D 0x0000004d +#define REG_A4XX_RBBM_POWER_CNTL_IP 0x00000098 +#define A4XX_RBBM_POWER_CNTL_IP_SW_COLLAPSE 0x00000001 +#define A4XX_RBBM_POWER_CNTL_IP_SP_TP_PWR_ON 0x00100000 + #define REG_A4XX_RBBM_PERFCTR_CP_0_LO 0x0000009c +#define REG_A4XX_RBBM_PERFCTR_CP_0_HI 0x0000009d + +#define REG_A4XX_RBBM_PERFCTR_CP_1_LO 0x0000009e + +#define REG_A4XX_RBBM_PERFCTR_CP_1_HI 0x0000009f + +#define REG_A4XX_RBBM_PERFCTR_CP_2_LO 0x000000a0 + +#define REG_A4XX_RBBM_PERFCTR_CP_2_HI 0x000000a1 + +#define REG_A4XX_RBBM_PERFCTR_CP_3_LO 0x000000a2 + +#define REG_A4XX_RBBM_PERFCTR_CP_3_HI 0x000000a3 + +#define REG_A4XX_RBBM_PERFCTR_CP_4_LO 0x000000a4 + +#define REG_A4XX_RBBM_PERFCTR_CP_4_HI 0x000000a5 + +#define REG_A4XX_RBBM_PERFCTR_CP_5_LO 0x000000a6 + +#define REG_A4XX_RBBM_PERFCTR_CP_5_HI 0x000000a7 + +#define REG_A4XX_RBBM_PERFCTR_CP_6_LO 0x000000a8 + +#define REG_A4XX_RBBM_PERFCTR_CP_6_HI 0x000000a9 + +#define REG_A4XX_RBBM_PERFCTR_CP_7_LO 0x000000aa + +#define REG_A4XX_RBBM_PERFCTR_CP_7_HI 0x000000ab + +#define REG_A4XX_RBBM_PERFCTR_RBBM_0_LO 0x000000ac + +#define REG_A4XX_RBBM_PERFCTR_RBBM_0_HI 0x000000ad + +#define REG_A4XX_RBBM_PERFCTR_RBBM_1_LO 0x000000ae + +#define REG_A4XX_RBBM_PERFCTR_RBBM_1_HI 0x000000af + +#define REG_A4XX_RBBM_PERFCTR_RBBM_2_LO 0x000000b0 + +#define REG_A4XX_RBBM_PERFCTR_RBBM_2_HI 0x000000b1 + +#define REG_A4XX_RBBM_PERFCTR_RBBM_3_LO 0x000000b2 + +#define REG_A4XX_RBBM_PERFCTR_RBBM_3_HI 0x000000b3 + +#define REG_A4XX_RBBM_PERFCTR_PC_0_LO 0x000000b4 + +#define REG_A4XX_RBBM_PERFCTR_PC_0_HI 0x000000b5 + +#define REG_A4XX_RBBM_PERFCTR_PC_1_LO 0x000000b6 + +#define REG_A4XX_RBBM_PERFCTR_PC_1_HI 0x000000b7 + +#define REG_A4XX_RBBM_PERFCTR_PC_2_LO 0x000000b8 + +#define REG_A4XX_RBBM_PERFCTR_PC_2_HI 0x000000b9 + +#define REG_A4XX_RBBM_PERFCTR_PC_3_LO 0x000000ba + +#define REG_A4XX_RBBM_PERFCTR_PC_3_HI 0x000000bb + +#define REG_A4XX_RBBM_PERFCTR_PC_4_LO 0x000000bc + +#define REG_A4XX_RBBM_PERFCTR_PC_4_HI 0x000000bd + +#define REG_A4XX_RBBM_PERFCTR_PC_5_LO 0x000000be + +#define REG_A4XX_RBBM_PERFCTR_PC_5_HI 0x000000bf + +#define REG_A4XX_RBBM_PERFCTR_PC_6_LO 0x000000c0 + +#define REG_A4XX_RBBM_PERFCTR_PC_6_HI 0x000000c1 + +#define REG_A4XX_RBBM_PERFCTR_PC_7_LO 0x000000c2 + +#define REG_A4XX_RBBM_PERFCTR_PC_7_HI 0x000000c3 + +#define REG_A4XX_RBBM_PERFCTR_VFD_0_LO 0x000000c4 + +#define REG_A4XX_RBBM_PERFCTR_VFD_0_HI 0x000000c5 + +#define REG_A4XX_RBBM_PERFCTR_VFD_1_LO 0x000000c6 + +#define REG_A4XX_RBBM_PERFCTR_VFD_1_HI 0x000000c7 + +#define REG_A4XX_RBBM_PERFCTR_VFD_2_LO 0x000000c8 + +#define REG_A4XX_RBBM_PERFCTR_VFD_2_HI 0x000000c9 + +#define REG_A4XX_RBBM_PERFCTR_VFD_3_LO 0x000000ca + +#define REG_A4XX_RBBM_PERFCTR_VFD_3_HI 0x000000cb + +#define REG_A4XX_RBBM_PERFCTR_VFD_4_LO 0x000000cc + +#define REG_A4XX_RBBM_PERFCTR_VFD_4_HI 0x000000cd + +#define REG_A4XX_RBBM_PERFCTR_VFD_5_LO 0x000000ce + +#define REG_A4XX_RBBM_PERFCTR_VFD_5_HI 0x000000cf + +#define REG_A4XX_RBBM_PERFCTR_VFD_6_LO 0x000000d0 + +#define REG_A4XX_RBBM_PERFCTR_VFD_6_HI 0x000000d1 + +#define REG_A4XX_RBBM_PERFCTR_VFD_7_LO 0x000000d2 + +#define REG_A4XX_RBBM_PERFCTR_VFD_7_HI 0x000000d3 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_0_LO 0x000000d4 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_0_HI 0x000000d5 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_1_LO 0x000000d6 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_1_HI 0x000000d7 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_2_LO 0x000000d8 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_2_HI 0x000000d9 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_3_LO 0x000000da + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_3_HI 0x000000db + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_4_LO 0x000000dc + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_4_HI 0x000000dd + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_5_LO 0x000000de + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_5_HI 0x000000df + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_6_LO 0x000000e0 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_6_HI 0x000000e1 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_7_LO 0x000000e2 + +#define REG_A4XX_RBBM_PERFCTR_HLSQ_7_HI 0x000000e3 + +#define REG_A4XX_RBBM_PERFCTR_VPC_0_LO 0x000000e4 + +#define REG_A4XX_RBBM_PERFCTR_VPC_0_HI 0x000000e5 + +#define REG_A4XX_RBBM_PERFCTR_VPC_1_LO 0x000000e6 + +#define REG_A4XX_RBBM_PERFCTR_VPC_1_HI 0x000000e7 + +#define REG_A4XX_RBBM_PERFCTR_VPC_2_LO 0x000000e8 + +#define REG_A4XX_RBBM_PERFCTR_VPC_2_HI 0x000000e9 + +#define REG_A4XX_RBBM_PERFCTR_VPC_3_LO 0x000000ea + +#define REG_A4XX_RBBM_PERFCTR_VPC_3_HI 0x000000eb + +#define REG_A4XX_RBBM_PERFCTR_CCU_0_LO 0x000000ec + +#define REG_A4XX_RBBM_PERFCTR_CCU_0_HI 0x000000ed + +#define REG_A4XX_RBBM_PERFCTR_CCU_1_LO 0x000000ee + +#define REG_A4XX_RBBM_PERFCTR_CCU_1_HI 0x000000ef + +#define REG_A4XX_RBBM_PERFCTR_CCU_2_LO 0x000000f0 + +#define REG_A4XX_RBBM_PERFCTR_CCU_2_HI 0x000000f1 + +#define REG_A4XX_RBBM_PERFCTR_CCU_3_LO 0x000000f2 + +#define REG_A4XX_RBBM_PERFCTR_CCU_3_HI 0x000000f3 + +#define REG_A4XX_RBBM_PERFCTR_TSE_0_LO 0x000000f4 + +#define REG_A4XX_RBBM_PERFCTR_TSE_0_HI 0x000000f5 + +#define REG_A4XX_RBBM_PERFCTR_TSE_1_LO 0x000000f6 + +#define REG_A4XX_RBBM_PERFCTR_TSE_1_HI 0x000000f7 + +#define REG_A4XX_RBBM_PERFCTR_TSE_2_LO 0x000000f8 + +#define REG_A4XX_RBBM_PERFCTR_TSE_2_HI 0x000000f9 + +#define REG_A4XX_RBBM_PERFCTR_TSE_3_LO 0x000000fa + +#define REG_A4XX_RBBM_PERFCTR_TSE_3_HI 0x000000fb + +#define REG_A4XX_RBBM_PERFCTR_RAS_0_LO 0x000000fc + +#define REG_A4XX_RBBM_PERFCTR_RAS_0_HI 0x000000fd + +#define REG_A4XX_RBBM_PERFCTR_RAS_1_LO 0x000000fe + +#define REG_A4XX_RBBM_PERFCTR_RAS_1_HI 0x000000ff + +#define REG_A4XX_RBBM_PERFCTR_RAS_2_LO 0x00000100 + +#define REG_A4XX_RBBM_PERFCTR_RAS_2_HI 0x00000101 + +#define REG_A4XX_RBBM_PERFCTR_RAS_3_LO 0x00000102 + +#define REG_A4XX_RBBM_PERFCTR_RAS_3_HI 0x00000103 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_0_LO 0x00000104 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_0_HI 0x00000105 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_1_LO 0x00000106 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_1_HI 0x00000107 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_2_LO 0x00000108 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_2_HI 0x00000109 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_3_LO 0x0000010a + +#define REG_A4XX_RBBM_PERFCTR_UCHE_3_HI 0x0000010b + +#define REG_A4XX_RBBM_PERFCTR_UCHE_4_LO 0x0000010c + +#define REG_A4XX_RBBM_PERFCTR_UCHE_4_HI 0x0000010d + +#define REG_A4XX_RBBM_PERFCTR_UCHE_5_LO 0x0000010e + +#define REG_A4XX_RBBM_PERFCTR_UCHE_5_HI 0x0000010f + +#define REG_A4XX_RBBM_PERFCTR_UCHE_6_LO 0x00000110 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_6_HI 0x00000111 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_7_LO 0x00000112 + +#define REG_A4XX_RBBM_PERFCTR_UCHE_7_HI 0x00000113 + +#define REG_A4XX_RBBM_PERFCTR_TP_0_LO 0x00000114 + +#define REG_A4XX_RBBM_PERFCTR_TP_0_HI 0x00000115 + +#define REG_A4XX_RBBM_PERFCTR_TP_0_LO 0x00000114 + +#define REG_A4XX_RBBM_PERFCTR_TP_0_HI 0x00000115 + +#define REG_A4XX_RBBM_PERFCTR_TP_1_LO 0x00000116 + +#define REG_A4XX_RBBM_PERFCTR_TP_1_HI 0x00000117 + +#define REG_A4XX_RBBM_PERFCTR_TP_2_LO 0x00000118 + +#define REG_A4XX_RBBM_PERFCTR_TP_2_HI 0x00000119 + +#define REG_A4XX_RBBM_PERFCTR_TP_3_LO 0x0000011a + +#define REG_A4XX_RBBM_PERFCTR_TP_3_HI 0x0000011b + +#define REG_A4XX_RBBM_PERFCTR_TP_4_LO 0x0000011c + +#define REG_A4XX_RBBM_PERFCTR_TP_4_HI 0x0000011d + +#define REG_A4XX_RBBM_PERFCTR_TP_5_LO 0x0000011e + +#define REG_A4XX_RBBM_PERFCTR_TP_5_HI 0x0000011f + +#define REG_A4XX_RBBM_PERFCTR_TP_6_LO 0x00000120 + +#define REG_A4XX_RBBM_PERFCTR_TP_6_HI 0x00000121 + +#define REG_A4XX_RBBM_PERFCTR_TP_7_LO 0x00000122 + +#define REG_A4XX_RBBM_PERFCTR_TP_7_HI 0x00000123 + +#define REG_A4XX_RBBM_PERFCTR_SP_0_LO 0x00000124 + +#define REG_A4XX_RBBM_PERFCTR_SP_0_HI 0x00000125 + +#define REG_A4XX_RBBM_PERFCTR_SP_1_LO 0x00000126 + +#define REG_A4XX_RBBM_PERFCTR_SP_1_HI 0x00000127 + +#define REG_A4XX_RBBM_PERFCTR_SP_2_LO 0x00000128 + +#define REG_A4XX_RBBM_PERFCTR_SP_2_HI 0x00000129 + +#define REG_A4XX_RBBM_PERFCTR_SP_3_LO 0x0000012a + +#define REG_A4XX_RBBM_PERFCTR_SP_3_HI 0x0000012b + +#define REG_A4XX_RBBM_PERFCTR_SP_4_LO 0x0000012c + +#define REG_A4XX_RBBM_PERFCTR_SP_4_HI 0x0000012d + +#define REG_A4XX_RBBM_PERFCTR_SP_5_LO 0x0000012e + +#define REG_A4XX_RBBM_PERFCTR_SP_5_HI 0x0000012f + +#define REG_A4XX_RBBM_PERFCTR_SP_6_LO 0x00000130 + +#define REG_A4XX_RBBM_PERFCTR_SP_6_HI 0x00000131 + +#define REG_A4XX_RBBM_PERFCTR_SP_7_LO 0x00000132 + +#define REG_A4XX_RBBM_PERFCTR_SP_7_HI 0x00000133 + +#define REG_A4XX_RBBM_PERFCTR_SP_8_LO 0x00000134 + +#define REG_A4XX_RBBM_PERFCTR_SP_8_HI 0x00000135 + +#define REG_A4XX_RBBM_PERFCTR_SP_9_LO 0x00000136 + +#define REG_A4XX_RBBM_PERFCTR_SP_9_HI 0x00000137 + +#define REG_A4XX_RBBM_PERFCTR_SP_10_LO 0x00000138 + +#define REG_A4XX_RBBM_PERFCTR_SP_10_HI 0x00000139 + +#define REG_A4XX_RBBM_PERFCTR_SP_11_LO 0x0000013a + +#define REG_A4XX_RBBM_PERFCTR_SP_11_HI 0x0000013b + +#define REG_A4XX_RBBM_PERFCTR_RB_0_LO 0x0000013c + +#define REG_A4XX_RBBM_PERFCTR_RB_0_HI 0x0000013d + +#define REG_A4XX_RBBM_PERFCTR_RB_1_LO 0x0000013e + +#define REG_A4XX_RBBM_PERFCTR_RB_1_HI 0x0000013f + +#define REG_A4XX_RBBM_PERFCTR_RB_2_LO 0x00000140 + +#define REG_A4XX_RBBM_PERFCTR_RB_2_HI 0x00000141 + +#define REG_A4XX_RBBM_PERFCTR_RB_3_LO 0x00000142 + +#define REG_A4XX_RBBM_PERFCTR_RB_3_HI 0x00000143 + +#define REG_A4XX_RBBM_PERFCTR_RB_4_LO 0x00000144 + +#define REG_A4XX_RBBM_PERFCTR_RB_4_HI 0x00000145 + +#define REG_A4XX_RBBM_PERFCTR_RB_5_LO 0x00000146 + +#define REG_A4XX_RBBM_PERFCTR_RB_5_HI 0x00000147 + +#define REG_A4XX_RBBM_PERFCTR_RB_6_LO 0x00000148 + +#define REG_A4XX_RBBM_PERFCTR_RB_6_HI 0x00000149 + +#define REG_A4XX_RBBM_PERFCTR_RB_7_LO 0x0000014a + +#define REG_A4XX_RBBM_PERFCTR_RB_7_HI 0x0000014b + +#define REG_A4XX_RBBM_PERFCTR_VSC_0_LO 0x0000014c + +#define REG_A4XX_RBBM_PERFCTR_VSC_0_HI 0x0000014d + +#define REG_A4XX_RBBM_PERFCTR_VSC_1_LO 0x0000014e + +#define REG_A4XX_RBBM_PERFCTR_VSC_1_HI 0x0000014f + +#define REG_A4XX_RBBM_PERFCTR_PWR_0_LO 0x00000166 + +#define REG_A4XX_RBBM_PERFCTR_PWR_0_HI 0x00000167 + +#define REG_A4XX_RBBM_PERFCTR_PWR_1_LO 0x00000168 + +#define REG_A4XX_RBBM_PERFCTR_PWR_1_HI 0x00000169 + +#define REG_A4XX_RBBM_ALWAYSON_COUNTER_LO 0x0000016e + +#define REG_A4XX_RBBM_ALWAYSON_COUNTER_HI 0x0000016f + static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL_SP(uint32_t i0) { return 0x00000068 + 0x1*i0; } static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL_SP_REG(uint32_t i0) { return 0x00000068 + 0x1*i0; } @@ -1046,6 +2061,10 @@ static inline uint32_t REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_REG(uint32_t i0) { return 0x0000008e + 0x1*i0; } +#define REG_A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_0 0x00000099 + +#define REG_A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_1 0x0000009a + #define REG_A4XX_RBBM_PERFCTR_PWR_1_LO 0x00000168 #define REG_A4XX_RBBM_PERFCTR_CTL 0x00000170 @@ -1060,6 +2079,14 @@ #define REG_A4XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x00000175 +#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_0 0x00000176 + +#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_1 0x00000177 + +#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_2 0x00000178 + +#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_3 0x00000179 + #define REG_A4XX_RBBM_GPU_BUSY_MASKED 0x0000017a #define REG_A4XX_RBBM_INT_0_STATUS 0x0000017d @@ -1099,6 +2126,11 @@ #define REG_A4XX_RBBM_INTERFACE_RRDY_STATUS5 0x0000019f +#define REG_A4XX_RBBM_POWER_STATUS 0x000001b0 +#define A4XX_RBBM_POWER_STATUS_SP_TP_PWR_ON 0x00100000 + +#define REG_A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL2 0x000001b8 + #define REG_A4XX_CP_SCRATCH_UMASK 0x00000228 #define REG_A4XX_CP_SCRATCH_ADDR 0x00000229 @@ -1167,11 +2199,23 @@ #define REG_A4XX_CP_DRAW_STATE_ADDR 0x00000232 -#define REG_A4XX_CP_PROTECT_REG_0 0x00000240 - static inline uint32_t REG_A4XX_CP_PROTECT(uint32_t i0) { return 0x00000240 + 0x1*i0; } static inline uint32_t REG_A4XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000240 + 0x1*i0; } +#define A4XX_CP_PROTECT_REG_BASE_ADDR__MASK 0x0001ffff +#define A4XX_CP_PROTECT_REG_BASE_ADDR__SHIFT 0 +static inline uint32_t A4XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val) +{ + return ((val) << A4XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A4XX_CP_PROTECT_REG_BASE_ADDR__MASK; +} +#define A4XX_CP_PROTECT_REG_MASK_LEN__MASK 0x1f000000 +#define A4XX_CP_PROTECT_REG_MASK_LEN__SHIFT 24 +static inline uint32_t A4XX_CP_PROTECT_REG_MASK_LEN(uint32_t val) +{ + return ((val) << A4XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A4XX_CP_PROTECT_REG_MASK_LEN__MASK; +} +#define A4XX_CP_PROTECT_REG_TRAP_WRITE 0x20000000 +#define A4XX_CP_PROTECT_REG_TRAP_READ 0x40000000 #define REG_A4XX_CP_PROTECT_CTRL 0x00000250 @@ -1191,6 +2235,20 @@ #define REG_A4XX_CP_PERFCTR_CP_SEL_0 0x00000500 +#define REG_A4XX_CP_PERFCTR_CP_SEL_1 0x00000501 + +#define REG_A4XX_CP_PERFCTR_CP_SEL_2 0x00000502 + +#define REG_A4XX_CP_PERFCTR_CP_SEL_3 0x00000503 + +#define REG_A4XX_CP_PERFCTR_CP_SEL_4 0x00000504 + +#define REG_A4XX_CP_PERFCTR_CP_SEL_5 0x00000505 + +#define REG_A4XX_CP_PERFCTR_CP_SEL_6 0x00000506 + +#define REG_A4XX_CP_PERFCTR_CP_SEL_7 0x00000507 + #define REG_A4XX_CP_PERFCOMBINER_SELECT 0x0000050b static inline uint32_t REG_A4XX_CP_SCRATCH(uint32_t i0) { return 0x00000578 + 0x1*i0; } @@ -1201,6 +2259,28 @@ #define REG_A4XX_SP_MODE_CONTROL 0x00000ec3 +#define REG_A4XX_SP_PERFCTR_SP_SEL_0 0x00000ec4 + +#define REG_A4XX_SP_PERFCTR_SP_SEL_1 0x00000ec5 + +#define REG_A4XX_SP_PERFCTR_SP_SEL_2 0x00000ec6 + +#define REG_A4XX_SP_PERFCTR_SP_SEL_3 0x00000ec7 + +#define REG_A4XX_SP_PERFCTR_SP_SEL_4 0x00000ec8 + +#define REG_A4XX_SP_PERFCTR_SP_SEL_5 0x00000ec9 + +#define REG_A4XX_SP_PERFCTR_SP_SEL_6 0x00000eca + +#define REG_A4XX_SP_PERFCTR_SP_SEL_7 0x00000ecb + +#define REG_A4XX_SP_PERFCTR_SP_SEL_8 0x00000ecc + +#define REG_A4XX_SP_PERFCTR_SP_SEL_9 0x00000ecd + +#define REG_A4XX_SP_PERFCTR_SP_SEL_10 0x00000ece + #define REG_A4XX_SP_PERFCTR_SP_SEL_11 0x00000ecf #define REG_A4XX_SP_SP_CTRL_REG 0x000022c0 @@ -1226,7 +2306,7 @@ { return ((val) << A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK; } -#define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0003fc00 +#define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0000fc00 #define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 10 static inline uint32_t A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) { @@ -1374,7 +2454,7 @@ { return ((val) << A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK; } -#define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0003fc00 +#define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0000fc00 #define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 10 static inline uint32_t A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) { @@ -1699,6 +2779,12 @@ #define REG_A4XX_VPC_DEBUG_ECO_CONTROL 0x00000e64 +#define REG_A4XX_VPC_PERFCTR_VPC_SEL_0 0x00000e65 + +#define REG_A4XX_VPC_PERFCTR_VPC_SEL_1 0x00000e66 + +#define REG_A4XX_VPC_PERFCTR_VPC_SEL_2 0x00000e67 + #define REG_A4XX_VPC_PERFCTR_VPC_SEL_3 0x00000e68 #define REG_A4XX_VPC_ATTR 0x00002140 @@ -1811,6 +2897,20 @@ #define REG_A4XX_VFD_DEBUG_CONTROL 0x00000e40 +#define REG_A4XX_VFD_PERFCTR_VFD_SEL_0 0x00000e43 + +#define REG_A4XX_VFD_PERFCTR_VFD_SEL_1 0x00000e44 + +#define REG_A4XX_VFD_PERFCTR_VFD_SEL_2 0x00000e45 + +#define REG_A4XX_VFD_PERFCTR_VFD_SEL_3 0x00000e46 + +#define REG_A4XX_VFD_PERFCTR_VFD_SEL_4 0x00000e47 + +#define REG_A4XX_VFD_PERFCTR_VFD_SEL_5 0x00000e48 + +#define REG_A4XX_VFD_PERFCTR_VFD_SEL_6 0x00000e49 + #define REG_A4XX_VFD_PERFCTR_VFD_SEL_7 0x00000e4a #define REG_A4XX_VGT_CL_INITIATOR 0x000021d0 @@ -1967,6 +3067,20 @@ #define REG_A4XX_TPL1_TP_MODE_CONTROL 0x00000f03 +#define REG_A4XX_TPL1_PERFCTR_TP_SEL_0 0x00000f04 + +#define REG_A4XX_TPL1_PERFCTR_TP_SEL_1 0x00000f05 + +#define REG_A4XX_TPL1_PERFCTR_TP_SEL_2 0x00000f06 + +#define REG_A4XX_TPL1_PERFCTR_TP_SEL_3 0x00000f07 + +#define REG_A4XX_TPL1_PERFCTR_TP_SEL_4 0x00000f08 + +#define REG_A4XX_TPL1_PERFCTR_TP_SEL_5 0x00000f09 + +#define REG_A4XX_TPL1_PERFCTR_TP_SEL_6 0x00000f0a + #define REG_A4XX_TPL1_PERFCTR_TP_SEL_7 0x00000f0b #define REG_A4XX_TPL1_TP_TEX_OFFSET 0x00002380 @@ -2021,9 +3135,25 @@ #define REG_A4XX_GRAS_PERFCTR_TSE_SEL_0 0x00000c88 +#define REG_A4XX_GRAS_PERFCTR_TSE_SEL_1 0x00000c89 + +#define REG_A4XX_GRAS_PERFCTR_TSE_SEL_2 0x00000c8a + #define REG_A4XX_GRAS_PERFCTR_TSE_SEL_3 0x00000c8b +#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_0 0x00000c8c + +#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_1 0x00000c8d + +#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_2 0x00000c8e + +#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_3 0x00000c8f + #define REG_A4XX_GRAS_CL_CLIP_CNTL 0x00002000 +#define A4XX_GRAS_CL_CLIP_CNTL_CLIP_DISABLE 0x00008000 +#define A4XX_GRAS_CL_CLIP_CNTL_ZNEAR_CLIP_DISABLE 0x00010000 +#define A4XX_GRAS_CL_CLIP_CNTL_ZFAR_CLIP_DISABLE 0x00020000 +#define A4XX_GRAS_CL_CLIP_CNTL_ZERO_GB_SCALE_Z 0x00400000 #define REG_A4XX_GRAS_CLEAR_CNTL 0x00002003 #define A4XX_GRAS_CLEAR_CNTL_NOT_FASTCLEAR 0x00000001 @@ -2114,6 +3244,7 @@ #define REG_A4XX_GRAS_ALPHA_CONTROL 0x00002073 #define A4XX_GRAS_ALPHA_CONTROL_ALPHA_TEST_ENABLE 0x00000004 +#define A4XX_GRAS_ALPHA_CONTROL_FORCE_FRAGZ_TO_FS 0x00000008 #define REG_A4XX_GRAS_SU_POLY_OFFSET_SCALE 0x00002074 #define A4XX_GRAS_SU_POLY_OFFSET_SCALE__MASK 0xffffffff @@ -2285,6 +3416,20 @@ #define REG_A4XX_UCHE_CACHE_WAYS_VFD 0x00000e8c +#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_0 0x00000e8e + +#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_1 0x00000e8f + +#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_2 0x00000e90 + +#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_3 0x00000e91 + +#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_4 0x00000e92 + +#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_5 0x00000e93 + +#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_6 0x00000e94 + #define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_7 0x00000e95 #define REG_A4XX_HLSQ_TIMEOUT_THRESHOLD 0x00000e00 @@ -2295,6 +3440,22 @@ #define REG_A4XX_HLSQ_PERF_PIPE_MASK 0x00000e0e +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_0 0x00000e06 + +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_1 0x00000e07 + +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_2 0x00000e08 + +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_3 0x00000e09 + +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_4 0x00000e0a + +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_5 0x00000e0b + +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_6 0x00000e0c + +#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_7 0x00000e0d + #define REG_A4XX_HLSQ_CONTROL_0_REG 0x000023c0 #define A4XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK 0x00000010 #define A4XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT 4 @@ -2545,14 +3706,42 @@ #define REG_A4XX_PC_BINNING_COMMAND 0x00000d00 #define A4XX_PC_BINNING_COMMAND_BINNING_ENABLE 0x00000001 +#define REG_A4XX_PC_TESSFACTOR_ADDR 0x00000d08 + #define REG_A4XX_PC_DRAWCALL_SETUP_OVERRIDE 0x00000d0c #define REG_A4XX_PC_PERFCTR_PC_SEL_0 0x00000d10 +#define REG_A4XX_PC_PERFCTR_PC_SEL_1 0x00000d11 + +#define REG_A4XX_PC_PERFCTR_PC_SEL_2 0x00000d12 + +#define REG_A4XX_PC_PERFCTR_PC_SEL_3 0x00000d13 + +#define REG_A4XX_PC_PERFCTR_PC_SEL_4 0x00000d14 + +#define REG_A4XX_PC_PERFCTR_PC_SEL_5 0x00000d15 + +#define REG_A4XX_PC_PERFCTR_PC_SEL_6 0x00000d16 + #define REG_A4XX_PC_PERFCTR_PC_SEL_7 0x00000d17 #define REG_A4XX_PC_BIN_BASE 0x000021c0 +#define REG_A4XX_PC_VSTREAM_CONTROL 0x000021c2 +#define A4XX_PC_VSTREAM_CONTROL_SIZE__MASK 0x003f0000 +#define A4XX_PC_VSTREAM_CONTROL_SIZE__SHIFT 16 +static inline uint32_t A4XX_PC_VSTREAM_CONTROL_SIZE(uint32_t val) +{ + return ((val) << A4XX_PC_VSTREAM_CONTROL_SIZE__SHIFT) & A4XX_PC_VSTREAM_CONTROL_SIZE__MASK; +} +#define A4XX_PC_VSTREAM_CONTROL_N__MASK 0x07c00000 +#define A4XX_PC_VSTREAM_CONTROL_N__SHIFT 22 +static inline uint32_t A4XX_PC_VSTREAM_CONTROL_N(uint32_t val) +{ + return ((val) << A4XX_PC_VSTREAM_CONTROL_N__SHIFT) & A4XX_PC_VSTREAM_CONTROL_N__MASK; +} + #define REG_A4XX_PC_PRIM_VTX_CNTL 0x000021c4 #define A4XX_PC_PRIM_VTX_CNTL_VAROUT__MASK 0x0000000f #define A4XX_PC_PRIM_VTX_CNTL_VAROUT__SHIFT 0 @@ -2564,7 +3753,20 @@ #define A4XX_PC_PRIM_VTX_CNTL_PROVOKING_VTX_LAST 0x02000000 #define A4XX_PC_PRIM_VTX_CNTL_PSIZE 0x04000000 -#define REG_A4XX_UNKNOWN_21C5 0x000021c5 +#define REG_A4XX_PC_PRIM_VTX_CNTL2 0x000021c5 +#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__MASK 0x00000007 +#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__SHIFT 0 +static inline uint32_t A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE(enum adreno_pa_su_sc_draw val) +{ + return ((val) << A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__SHIFT) & A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__MASK; +} +#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__MASK 0x00000038 +#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__SHIFT 3 +static inline uint32_t A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE(enum adreno_pa_su_sc_draw val) +{ + return ((val) << A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__SHIFT) & A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__MASK; +} +#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_ENABLE 0x00000040 #define REG_A4XX_PC_RESTART_INDEX 0x000021c6 @@ -2602,12 +3804,8 @@ { return ((val) << A4XX_PC_HS_PARAM_SPACING__SHIFT) & A4XX_PC_HS_PARAM_SPACING__MASK; } -#define A4XX_PC_HS_PARAM_PRIMTYPE__MASK 0x01800000 -#define A4XX_PC_HS_PARAM_PRIMTYPE__SHIFT 23 -static inline uint32_t A4XX_PC_HS_PARAM_PRIMTYPE(enum adreno_pa_su_sc_draw val) -{ - return ((val) << A4XX_PC_HS_PARAM_PRIMTYPE__SHIFT) & A4XX_PC_HS_PARAM_PRIMTYPE__MASK; -} +#define A4XX_PC_HS_PARAM_CW 0x00800000 +#define A4XX_PC_HS_PARAM_CONNECTED 0x01000000 #define REG_A4XX_VBIF_VERSION 0x00003000 @@ -2646,20 +3844,6 @@ #define REG_A4XX_UNKNOWN_20EF 0x000020ef -#define REG_A4XX_UNKNOWN_20F0 0x000020f0 - -#define REG_A4XX_UNKNOWN_20F1 0x000020f1 - -#define REG_A4XX_UNKNOWN_20F2 0x000020f2 - -#define REG_A4XX_UNKNOWN_20F7 0x000020f7 -#define A4XX_UNKNOWN_20F7__MASK 0xffffffff -#define A4XX_UNKNOWN_20F7__SHIFT 0 -static inline uint32_t A4XX_UNKNOWN_20F7(float val) -{ - return ((fui(val)) << A4XX_UNKNOWN_20F7__SHIFT) & A4XX_UNKNOWN_20F7__MASK; -} - #define REG_A4XX_UNKNOWN_2152 0x00002152 #define REG_A4XX_UNKNOWN_2153 0x00002153 @@ -2720,6 +3904,12 @@ { return ((val) << A4XX_TEX_SAMP_0_ANISO__SHIFT) & A4XX_TEX_SAMP_0_ANISO__MASK; } +#define A4XX_TEX_SAMP_0_LOD_BIAS__MASK 0xfff80000 +#define A4XX_TEX_SAMP_0_LOD_BIAS__SHIFT 19 +static inline uint32_t A4XX_TEX_SAMP_0_LOD_BIAS(float val) +{ + return ((((int32_t)(val * 256.0))) << A4XX_TEX_SAMP_0_LOD_BIAS__SHIFT) & A4XX_TEX_SAMP_0_LOD_BIAS__MASK; +} #define REG_A4XX_TEX_SAMP_1 0x00000001 #define A4XX_TEX_SAMP_1_COMPARE_FUNC__MASK 0x0000000e @@ -2728,6 +3918,7 @@ { return ((val) << A4XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT) & A4XX_TEX_SAMP_1_COMPARE_FUNC__MASK; } +#define A4XX_TEX_SAMP_1_CUBEMAPSEAMLESSFILTOFF 0x00000010 #define A4XX_TEX_SAMP_1_UNNORM_COORDS 0x00000020 #define A4XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR 0x00000040 #define A4XX_TEX_SAMP_1_MAX_LOD__MASK 0x000fff00 @@ -2796,7 +3987,7 @@ { return ((val) << A4XX_TEX_CONST_1_HEIGHT__SHIFT) & A4XX_TEX_CONST_1_HEIGHT__MASK; } -#define A4XX_TEX_CONST_1_WIDTH__MASK 0x1fff8000 +#define A4XX_TEX_CONST_1_WIDTH__MASK 0x3fff8000 #define A4XX_TEX_CONST_1_WIDTH__SHIFT 15 static inline uint32_t A4XX_TEX_CONST_1_WIDTH(uint32_t val) { diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_common.xml.h linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_common.xml.h --- linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_common.xml.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_common.xml.h 2019-01-22 16:16:23.483246225 +0100 @@ -8,17 +8,19 @@ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00) +- ./adreno.xml ( 431 bytes, from 2016-10-24 21:12:27) +- ./freedreno_copyright.xml ( 1572 bytes, from 2016-10-24 21:12:27) +- ./adreno/a2xx.xml ( 32901 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_common.xml ( 12025 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_pm4.xml ( 19684 bytes, from 2016-10-24 21:12:27) +- ./adreno/a3xx.xml ( 83840 bytes, from 2016-10-24 21:12:27) +- ./adreno/a4xx.xml ( 110708 bytes, from 2016-10-24 21:12:27) +- ./adreno/a5xx.xml ( 81207 bytes, from 2016-10-26 19:36:59) +- ./adreno/ocmem.xml ( 1773 bytes, from 2016-10-24 21:12:27) -Copyright (C) 2013-2015 by the following authors: +Copyright (C) 2013-2016 by the following authors: - Rob Clark (robclark) +- Ilia Mirkin (imirkin) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -119,6 +121,25 @@ RB_COPY_DEPTH_STENCIL = 5, }; +enum a3xx_rop_code { + ROP_CLEAR = 0, + ROP_NOR = 1, + ROP_AND_INVERTED = 2, + ROP_COPY_INVERTED = 3, + ROP_AND_REVERSE = 4, + ROP_INVERT = 5, + ROP_XOR = 6, + ROP_NAND = 7, + ROP_AND = 8, + ROP_EQUIV = 9, + ROP_NOOP = 10, + ROP_OR_INVERTED = 11, + ROP_COPY = 12, + ROP_OR_REVERSE = 13, + ROP_OR = 14, + ROP_SET = 15, +}; + enum a3xx_render_mode { RB_RENDERING_PASS = 0, RB_TILING_PASS = 1, @@ -154,6 +175,14 @@ XYZW = 3, }; +enum a3xx_rb_blend_opcode { + BLEND_DST_PLUS_SRC = 0, + BLEND_SRC_MINUS_DST = 1, + BLEND_DST_MINUS_SRC = 2, + BLEND_MIN_DST_SRC = 3, + BLEND_MAX_DST_SRC = 4, +}; + #define REG_AXXX_CP_RB_BASE 0x000001c0 #define REG_AXXX_CP_RB_CNTL 0x000001c1 diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_device.c linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_device.c --- linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_device.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_device.c 2019-01-22 16:16:23.487246262 +0100 @@ -19,10 +19,6 @@ #include "adreno_gpu.h" -#if defined(DOWNSTREAM_CONFIG_MSM_BUS_SCALING) && !defined(CONFIG_OF) -# include -#endif - #define ANY_ID 0xff bool hang_debug = false; @@ -31,6 +27,7 @@ struct msm_gpu *a3xx_gpu_init(struct drm_device *dev); struct msm_gpu *a4xx_gpu_init(struct drm_device *dev); +struct msm_gpu *a5xx_gpu_init(struct drm_device *dev); static const struct adreno_info gpulist[] = { { @@ -73,6 +70,30 @@ .pfpfw = "a420_pfp.fw", .gmem = (SZ_1M + SZ_512K), .init = a4xx_gpu_init, + }, { + .rev = ADRENO_REV(4, 3, 0, ANY_ID), + .revn = 430, + .name = "A430", + .pm4fw = "a420_pm4.fw", + .pfpfw = "a420_pfp.fw", + .gmem = (SZ_1M + SZ_512K), + .init = a4xx_gpu_init, + }, { + .rev = ADRENO_REV(5, 3, 0, ANY_ID), + .revn = 530, + .name = "A530", + .pm4fw = "a530_pm4.fw", + .pfpfw = "a530_pfp.fw", + .gmem = SZ_1M, + .init = a5xx_gpu_init, + }, { + .rev = ADRENO_REV(5, 4, 0, ANY_ID), + .revn = 540, + .name = "A540", + .pm4fw = "a530_pm4.fw", + .pfpfw = "a530_pfp.fw", + .gmem = SZ_1M, + .init = a5xx_gpu_init, }, }; @@ -82,6 +103,8 @@ MODULE_FIRMWARE("a330_pfp.fw"); MODULE_FIRMWARE("a420_pm4.fw"); MODULE_FIRMWARE("a420_pfp.fw"); +MODULE_FIRMWARE("a530_fm4.fw"); +MODULE_FIRMWARE("a530_pfp.fw"); static inline bool _rev_match(uint8_t entry, uint8_t id) { @@ -141,17 +164,17 @@ if (gpu) { int ret; - mutex_lock(&dev->struct_mutex); - gpu->funcs->pm_resume(gpu); - mutex_unlock(&dev->struct_mutex); - ret = gpu->funcs->hw_init(gpu); + + pm_runtime_get_sync(&pdev->dev); + ret = msm_gpu_hw_init(gpu); + pm_runtime_put_sync_autosuspend(&pdev->dev); if (ret) { dev_err(dev->dev, "gpu hw init failed: %d\n", ret); + mutex_lock(&dev->struct_mutex); + gpu->funcs->pm_suspend(gpu); + mutex_unlock(&dev->struct_mutex); gpu->funcs->destroy(gpu); gpu = NULL; - } else { - /* give inactive pm a chance to kick in: */ - msm_gpu_retire(gpu); } } @@ -168,12 +191,16 @@ static int adreno_bind(struct device *dev, struct device *master, void *data) { static struct adreno_platform_config config = {}; -#ifdef CONFIG_OF - struct device_node *child, *node = dev->of_node; - u32 val; + uint32_t val = 0; int ret; - ret = of_property_read_u32(node, "qcom,chipid", &val); + /* + * Read the chip ID from the device tree at bind time - we use this + * information to load the correct functions. All the rest of the + * (extensive) device tree probing should happen in the GPU specific + * code + */ + ret = of_property_read_u32(dev->of_node, "qcom,chipid", &val); if (ret) { dev_err(dev, "could not find chipid: %d\n", ret); return ret; @@ -182,76 +209,6 @@ config.rev = ADRENO_REV((val >> 24) & 0xff, (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff); - /* find clock rates: */ - config.fast_rate = 0; - config.slow_rate = ~0; - for_each_child_of_node(node, child) { - if (of_device_is_compatible(child, "qcom,gpu-pwrlevels")) { - struct device_node *pwrlvl; - for_each_child_of_node(child, pwrlvl) { - ret = of_property_read_u32(pwrlvl, "qcom,gpu-freq", &val); - if (ret) { - dev_err(dev, "could not find gpu-freq: %d\n", ret); - return ret; - } - config.fast_rate = max(config.fast_rate, val); - config.slow_rate = min(config.slow_rate, val); - } - } - } - - if (!config.fast_rate) { - dev_err(dev, "could not find clk rates\n"); - return -ENXIO; - } - -#else - struct kgsl_device_platform_data *pdata = dev->platform_data; - uint32_t version = socinfo_get_version(); - if (cpu_is_apq8064ab()) { - config.fast_rate = 450000000; - config.slow_rate = 27000000; - config.bus_freq = 4; - config.rev = ADRENO_REV(3, 2, 1, 0); - } else if (cpu_is_apq8064()) { - config.fast_rate = 400000000; - config.slow_rate = 27000000; - config.bus_freq = 4; - - if (SOCINFO_VERSION_MAJOR(version) == 2) - config.rev = ADRENO_REV(3, 2, 0, 2); - else if ((SOCINFO_VERSION_MAJOR(version) == 1) && - (SOCINFO_VERSION_MINOR(version) == 1)) - config.rev = ADRENO_REV(3, 2, 0, 1); - else - config.rev = ADRENO_REV(3, 2, 0, 0); - - } else if (cpu_is_msm8960ab()) { - config.fast_rate = 400000000; - config.slow_rate = 320000000; - config.bus_freq = 4; - - if (SOCINFO_VERSION_MINOR(version) == 0) - config.rev = ADRENO_REV(3, 2, 1, 0); - else - config.rev = ADRENO_REV(3, 2, 1, 1); - - } else if (cpu_is_msm8930()) { - config.fast_rate = 400000000; - config.slow_rate = 27000000; - config.bus_freq = 3; - - if ((SOCINFO_VERSION_MAJOR(version) == 1) && - (SOCINFO_VERSION_MINOR(version) == 2)) - config.rev = ADRENO_REV(3, 0, 5, 2); - else - config.rev = ADRENO_REV(3, 0, 5, 0); - - } -# ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING - config.bus_scale_table = pdata->bus_scale_table; -# endif -#endif dev->platform_data = &config; set_gpu_pdev(dev_get_drvdata(master), to_platform_device(dev)); return 0; @@ -286,12 +243,35 @@ {} }; +#ifdef CONFIG_PM +static int adreno_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct msm_gpu *gpu = platform_get_drvdata(pdev); + + return gpu->funcs->pm_resume(gpu); +} + +static int adreno_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct msm_gpu *gpu = platform_get_drvdata(pdev); + + return gpu->funcs->pm_suspend(gpu); +} +#endif + +static const struct dev_pm_ops adreno_pm_ops = { + SET_RUNTIME_PM_OPS(adreno_suspend, adreno_resume, NULL) +}; + static struct platform_driver adreno_driver = { .probe = adreno_probe, .remove = adreno_remove, .driver = { .name = "adreno", .of_match_table = dt_match, + .pm = &adreno_pm_ops, }, }; diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_gpu.c linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_gpu.c --- linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_gpu.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_gpu.c 2019-01-22 16:16:23.487246262 +0100 @@ -2,7 +2,7 @@ * Copyright (C) 2013 Red Hat * Author: Rob Clark * - * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (c) 2014,2016-2018 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published by @@ -17,12 +17,12 @@ * this program. If not, see . */ +#include #include "adreno_gpu.h" +#include "msm_snapshot.h" #include "msm_gem.h" #include "msm_mmu.h" -#define RB_SIZE SZ_32K -#define RB_BLKSIZE 16 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value) { @@ -35,123 +35,167 @@ case MSM_PARAM_GMEM_SIZE: *value = adreno_gpu->gmem; return 0; + case MSM_PARAM_GMEM_BASE: + *value = 0x100000; + return 0; case MSM_PARAM_CHIP_ID: *value = adreno_gpu->rev.patchid | (adreno_gpu->rev.minor << 8) | (adreno_gpu->rev.major << 16) | (adreno_gpu->rev.core << 24); return 0; + case MSM_PARAM_MAX_FREQ: + *value = gpu->gpufreq[0]; + return 0; + case MSM_PARAM_TIMESTAMP: + if (adreno_gpu->funcs->get_timestamp) { + int ret; + + pm_runtime_get_sync(&gpu->pdev->dev); + ret = adreno_gpu->funcs->get_timestamp(gpu, value); + pm_runtime_put_autosuspend(&gpu->pdev->dev); + + return ret; + } + return -EINVAL; + case MSM_PARAM_NR_RINGS: + *value = gpu->nr_rings; + return 0; + case MSM_PARAM_GPU_HANG_TIMEOUT: + *value = DRM_MSM_HANGCHECK_PERIOD; + return 0; default: DBG("%s: invalid param: %u", gpu->name, param); return -EINVAL; } } -#define rbmemptr(adreno_gpu, member) \ - ((adreno_gpu)->memptrs_iova + offsetof(struct adreno_rbmemptrs, member)) - int adreno_hw_init(struct msm_gpu *gpu) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); - int ret; + int i; DBG("%s", gpu->name); - ret = msm_gem_get_iova(gpu->rb->bo, gpu->id, &gpu->rb_iova); + for (i = 0; i < gpu->nr_rings; i++) { + struct msm_ringbuffer *ring = gpu->rb[i]; + + int ret = msm_gem_get_iova(ring->bo, gpu->aspace, + &ring->iova); if (ret) { - gpu->rb_iova = 0; - dev_err(gpu->dev->dev, "could not map ringbuffer: %d\n", ret); + ring->iova = 0; + dev_err(gpu->dev->dev, + "could not map ringbuffer %d: %d\n", i, ret); return ret; } - /* Setup REG_CP_RB_CNTL: */ - adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_CNTL, - /* size is log2(quad-words): */ - AXXX_CP_RB_CNTL_BUFSZ(ilog2(gpu->rb->size / 8)) | - AXXX_CP_RB_CNTL_BLKSZ(ilog2(RB_BLKSIZE / 8))); + /* reset ringbuffer(s): */ + /* No need for a lock here, nobody else is peeking in */ + ring->cur = ring->start; + ring->next = ring->start; + + /* reset completed fence seqno, discard anything pending: */ + ring->memptrs->fence = adreno_submitted_fence(gpu, ring); + ring->memptrs->rptr = 0; + } - /* Setup ringbuffer address: */ - adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_BASE, gpu->rb_iova); - adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_RPTR_ADDR, - rbmemptr(adreno_gpu, rptr)); + /* + * Setup REG_CP_RB_CNTL. The same value is used across targets (with + * the excpetion of A430 that disables the RPTR shadow) - the cacluation + * for the ringbuffer size and block size is moved to msm_gpu.h for the + * pre-processor to deal with and the A430 variant is ORed in here + */ + adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_CNTL, + MSM_GPU_RB_CNTL_DEFAULT | + (adreno_is_a430(adreno_gpu) ? AXXX_CP_RB_CNTL_NO_UPDATE : 0)); - /* Setup scratch/timestamp: */ - adreno_gpu_write(adreno_gpu, REG_ADRENO_SCRATCH_ADDR, - rbmemptr(adreno_gpu, fence)); + /* Setup ringbuffer address - use ringbuffer[0] for GPU init */ + adreno_gpu_write64(adreno_gpu, REG_ADRENO_CP_RB_BASE, + REG_ADRENO_CP_RB_BASE_HI, gpu->rb[0]->iova); - adreno_gpu_write(adreno_gpu, REG_ADRENO_SCRATCH_UMSK, 0x1); + adreno_gpu_write64(adreno_gpu, REG_ADRENO_CP_RB_RPTR_ADDR, + REG_ADRENO_CP_RB_RPTR_ADDR_HI, rbmemptr(gpu->rb[0], rptr)); return 0; } -static uint32_t get_wptr(struct msm_ringbuffer *ring) +/* Use this helper to read rptr, since a430 doesn't update rptr in memory */ +static uint32_t get_rptr(struct adreno_gpu *adreno_gpu, + struct msm_ringbuffer *ring) { - return ring->cur - ring->start; + if (adreno_is_a430(adreno_gpu)) { + /* + * If index is anything but 0 this will probably break horribly, + * but I think that we have enough infrastructure in place to + * ensure that it won't be. If not then this is why your + * a430 stopped working. + */ + return ring->memptrs->rptr = + adreno_gpu_read(adreno_gpu, REG_ADRENO_CP_RB_RPTR); + } + + return ring->memptrs->rptr; } -uint32_t adreno_last_fence(struct msm_gpu *gpu) +struct msm_ringbuffer *adreno_active_ring(struct msm_gpu *gpu) { - struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); - return adreno_gpu->memptrs->fence; + return gpu->rb[0]; +} + +uint32_t adreno_submitted_fence(struct msm_gpu *gpu, + struct msm_ringbuffer *ring) +{ + if (!ring) + return 0; + + return ring->submitted_fence; } void adreno_recover(struct msm_gpu *gpu) { - struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct drm_device *dev = gpu->dev; int ret; - gpu->funcs->pm_suspend(gpu); - - /* reset ringbuffer: */ - gpu->rb->cur = gpu->rb->start; + /* + * XXX pm-runtime?? we *need* the device to be off after this + * so maybe continuing to call ->pm_suspend/resume() is better? + */ - /* reset completed fence seqno, just discard anything pending: */ - adreno_gpu->memptrs->fence = gpu->submitted_fence; - adreno_gpu->memptrs->rptr = 0; - adreno_gpu->memptrs->wptr = 0; + gpu->funcs->pm_suspend(gpu); gpu->funcs->pm_resume(gpu); - ret = gpu->funcs->hw_init(gpu); + + ret = msm_gpu_hw_init(gpu); if (ret) { dev_err(dev->dev, "gpu hw init failed: %d\n", ret); /* hmm, oh well? */ } } -int adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, - struct msm_file_private *ctx) +void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); - struct msm_drm_private *priv = gpu->dev->dev_private; - struct msm_ringbuffer *ring = gpu->rb; - unsigned i, ibs = 0; + struct msm_ringbuffer *ring = gpu->rb[submit->ring]; + unsigned i; for (i = 0; i < submit->nr_cmds; i++) { switch (submit->cmd[i].type) { case MSM_SUBMIT_CMD_IB_TARGET_BUF: /* ignore IB-targets */ break; + case MSM_SUBMIT_CMD_PROFILE_BUF: case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: - /* ignore if there has not been a ctx switch: */ - if (priv->lastctx == ctx) break; case MSM_SUBMIT_CMD_BUF: - OUT_PKT3(ring, CP_INDIRECT_BUFFER_PFD, 2); - OUT_RING(ring, submit->cmd[i].iova); + OUT_PKT3(ring, adreno_is_a430(adreno_gpu) ? + CP_INDIRECT_BUFFER_PFE : CP_INDIRECT_BUFFER_PFD, 2); + OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); OUT_RING(ring, submit->cmd[i].size); - ibs++; + OUT_PKT2(ring); break; } } - /* on a320, at least, we seem to need to pad things out to an - * even number of qwords to avoid issue w/ CP hanging on wrap- - * around: - */ - if (ibs % 2) - OUT_PKT2(ring); - OUT_PKT0(ring, REG_AXXX_CP_SCRATCH_REG2, 1); OUT_RING(ring, submit->fence); @@ -169,7 +213,7 @@ OUT_PKT3(ring, CP_EVENT_WRITE, 3); OUT_RING(ring, CACHE_FLUSH_TS); - OUT_RING(ring, rbmemptr(adreno_gpu, fence)); + OUT_RING(ring, rbmemptr(ring, fence)); OUT_RING(ring, submit->fence); /* we could maybe be clever and only CP_COND_EXEC the interrupt: */ @@ -196,22 +240,23 @@ } #endif - gpu->funcs->flush(gpu); - - return 0; + gpu->funcs->flush(gpu, ring); } -void adreno_flush(struct msm_gpu *gpu) +void adreno_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); uint32_t wptr; + /* Copy the shadow to the actual register */ + ring->cur = ring->next; + /* - * Mask wptr value that we calculate to fit in the HW range. This is + * Mask the wptr value that we calculate to fit in the HW range. This is * to account for the possibility that the last command fit exactly into * the ringbuffer and rb->next hasn't wrapped to zero yet */ - wptr = get_wptr(gpu->rb) & ((gpu->rb->size / 4) - 1); + wptr = get_wptr(ring); /* ensure writes to ringbuffer have hit system memory: */ mb(); @@ -219,22 +264,27 @@ adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_WPTR, wptr); } -void adreno_idle(struct msm_gpu *gpu) +bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); - uint32_t wptr = get_wptr(gpu->rb); + uint32_t wptr = get_wptr(ring); /* wait for CP to drain ringbuffer: */ - if (spin_until(adreno_gpu->memptrs->rptr == wptr)) - DRM_ERROR("%s: timeout waiting to drain ringbuffer!\n", gpu->name); + if (!spin_until(get_rptr(adreno_gpu, ring) == wptr)) + return true; /* TODO maybe we need to reset GPU here to recover from hang? */ + DRM_ERROR("%s: timeout waiting to drain ringbuffer %d rptr/wptr = %X/%X\n", + gpu->name, ring->id, get_rptr(adreno_gpu, ring), wptr); + + return false; } #ifdef CONFIG_DEBUG_FS void adreno_show(struct msm_gpu *gpu, struct seq_file *m) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct msm_ringbuffer *ring; int i; seq_printf(m, "revision: %d (%d.%d.%d.%d)\n", @@ -242,13 +292,18 @@ adreno_gpu->rev.major, adreno_gpu->rev.minor, adreno_gpu->rev.patchid); - seq_printf(m, "fence: %d/%d\n", adreno_gpu->memptrs->fence, - gpu->submitted_fence); - seq_printf(m, "rptr: %d\n", adreno_gpu->memptrs->rptr); - seq_printf(m, "wptr: %d\n", adreno_gpu->memptrs->wptr); - seq_printf(m, "rb wptr: %d\n", get_wptr(gpu->rb)); - - gpu->funcs->pm_resume(gpu); + FOR_EACH_RING(gpu, ring, i) { + if (!ring) + continue; + + seq_printf(m, "rb %d: fence: %d/%d\n", i, + ring->memptrs->fence, + adreno_submitted_fence(gpu, ring)); + + seq_printf(m, " rptr: %d\n", + get_rptr(adreno_gpu, ring)); + seq_printf(m, "rb wptr: %d\n", get_wptr(ring)); + } /* dump these out in a form that can be parsed by demsm: */ seq_printf(m, "IO:region %s 00000000 00020000\n", gpu->name); @@ -262,8 +317,6 @@ seq_printf(m, "IO:R %08x %08x\n", addr<<2, val); } } - - gpu->funcs->pm_suspend(gpu); } #endif @@ -275,22 +328,29 @@ */ void adreno_dump_info(struct msm_gpu *gpu) { + struct drm_device *dev = gpu->dev; struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct msm_ringbuffer *ring; int i; - printk("revision: %d (%d.%d.%d.%d)\n", + dev_err(dev->dev, "revision: %d (%d.%d.%d.%d)\n", adreno_gpu->info->revn, adreno_gpu->rev.core, adreno_gpu->rev.major, adreno_gpu->rev.minor, adreno_gpu->rev.patchid); - printk("fence: %d/%d\n", adreno_gpu->memptrs->fence, - gpu->submitted_fence); - printk("rptr: %d\n", adreno_gpu->memptrs->rptr); - printk("wptr: %d\n", adreno_gpu->memptrs->wptr); - printk("rb wptr: %d\n", get_wptr(gpu->rb)); + FOR_EACH_RING(gpu, ring, i) { + if (!ring) + continue; + + dev_err(dev->dev, " ring %d: fence %d/%d rptr/wptr %x/%x\n", i, + ring->memptrs->fence, + adreno_submitted_fence(gpu, ring), + get_rptr(adreno_gpu, ring), + get_wptr(ring)); + } for (i = 0; i < 8; i++) { - printk("CP_SCRATCH_REG%d: %u\n", i, + pr_err("CP_SCRATCH_REG%d: %u\n", i, gpu_read(gpu, REG_AXXX_CP_SCRATCH_REG0 + i)); } } @@ -315,32 +375,135 @@ } } -static uint32_t ring_freewords(struct msm_gpu *gpu) +static uint32_t ring_freewords(struct msm_ringbuffer *ring) { - struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); - uint32_t size = gpu->rb->size / 4; - uint32_t wptr = get_wptr(gpu->rb); - uint32_t rptr = adreno_gpu->memptrs->rptr; + struct adreno_gpu *adreno_gpu = to_adreno_gpu(ring->gpu); + uint32_t size = MSM_GPU_RINGBUFFER_SZ >> 2; + /* Use ring->next to calculate free size */ + uint32_t wptr = ring->next - ring->start; + uint32_t rptr = get_rptr(adreno_gpu, ring); return (rptr + (size - 1) - wptr) % size; } -void adreno_wait_ring(struct msm_gpu *gpu, uint32_t ndwords) +void adreno_wait_ring(struct msm_ringbuffer *ring, uint32_t ndwords) { - if (spin_until(ring_freewords(gpu) >= ndwords)) - DRM_ERROR("%s: timeout waiting for ringbuffer space\n", gpu->name); + if (spin_until(ring_freewords(ring) >= ndwords)) + DRM_ERROR("%s: timeout waiting for space in ringubffer %d\n", + ring->gpu->name, ring->id); } -static const char *iommu_ports[] = { - "gfx3d_user", "gfx3d_priv", - "gfx3d1_user", "gfx3d1_priv", +/* Read the set of powerlevels */ +static int _adreno_get_pwrlevels(struct msm_gpu *gpu, struct device_node *node) +{ + struct device_node *child; + + for_each_child_of_node(node, child) { + unsigned int index; + + if (of_property_read_u32(child, "reg", &index)) + return -EINVAL; + + if (index >= ARRAY_SIZE(gpu->gpufreq)) + continue; + + gpu->nr_pwrlevels = max(gpu->nr_pwrlevels, index + 1); + + of_property_read_u32(child, "qcom,gpu-freq", + &gpu->gpufreq[index]); + of_property_read_u32(child, "qcom,bus-freq", + &gpu->busfreq[index]); + } + + DBG("fast_rate=%u, slow_rate=%u, bus_freq=%u", + gpu->gpufreq[0], + gpu->gpufreq[gpu->nr_pwrlevels - 1], + gpu->busfreq[0]); + + return 0; +} + +/* + * Escape valve for targets that don't define the binning nodes. Get the + * first powerlevel node and parse it + */ +static int adreno_get_legacy_pwrlevels(struct msm_gpu *gpu, + struct device_node *parent) +{ + struct device_node *child; + + child = of_find_node_by_name(parent, "qcom,gpu-pwrlevels"); + if (child) + return _adreno_get_pwrlevels(gpu, child); + + dev_err(gpu->dev->dev, "Unable to parse any powerlevels\n"); + return -EINVAL; +} + +/* Get the powerlevels for the target */ +static int adreno_get_pwrlevels(struct msm_gpu *gpu, struct device_node *parent) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct device_node *node, *child; + + /* See if the target has defined a number of power bins */ + node = of_find_node_by_name(parent, "qcom,gpu-pwrlevel-bins"); + if (!node) { + /* If not look for the qcom,gpu-pwrlevels node */ + return adreno_get_legacy_pwrlevels(gpu, parent); + } + + for_each_child_of_node(node, child) { + unsigned int bin; + + if (of_property_read_u32(child, "qcom,speed-bin", &bin)) + continue; + + /* + * If the bin matches the bin specified by the fuses, then we + * have a winner - parse it + */ + if (adreno_gpu->speed_bin == bin) + return _adreno_get_pwrlevels(gpu, child); + } + + return -ENODEV; +} + +static const struct { + const char *str; + uint32_t flag; +} quirks[] = { + { "qcom,gpu-quirk-two-pass-use-wfi", ADRENO_QUIRK_TWO_PASS_USE_WFI }, + { "qcom,gpu-quirk-fault-detect-mask", ADRENO_QUIRK_FAULT_DETECT_MASK }, }; +/* Parse the statistics from the device tree */ +static int adreno_of_parse(struct platform_device *pdev, struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct device_node *node = pdev->dev.of_node; + int i, ret; + + /* Probe the powerlevels */ + ret = adreno_get_pwrlevels(gpu, node); + if (ret) + return ret; + + /* Check to see if any quirks were specified in the device tree */ + for (i = 0; i < ARRAY_SIZE(quirks); i++) + if (of_property_read_bool(node, quirks[i].str)) + adreno_gpu->quirks |= quirks[i].flag; + + return 0; +} + int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev, - struct adreno_gpu *adreno_gpu, const struct adreno_gpu_funcs *funcs) + struct adreno_gpu *adreno_gpu, + const struct adreno_gpu_funcs *funcs, + struct msm_gpu_config *gpu_config) { struct adreno_platform_config *config = pdev->dev.platform_data; struct msm_gpu *gpu = &adreno_gpu->base; - struct msm_mmu *mmu; int ret; adreno_gpu->funcs = funcs; @@ -349,22 +512,18 @@ adreno_gpu->revn = adreno_gpu->info->revn; adreno_gpu->rev = config->rev; - gpu->fast_rate = config->fast_rate; - gpu->slow_rate = config->slow_rate; - gpu->bus_freq = config->bus_freq; -#ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING - gpu->bus_scale_table = config->bus_scale_table; -#endif - - DBG("fast_rate=%u, slow_rate=%u, bus_freq=%u", - gpu->fast_rate, gpu->slow_rate, gpu->bus_freq); + /* Get the rest of the target configuration from the device tree */ + adreno_of_parse(pdev, gpu); ret = msm_gpu_init(drm, pdev, &adreno_gpu->base, &funcs->base, - adreno_gpu->info->name, "kgsl_3d0_reg_memory", "kgsl_3d0_irq", - RB_SIZE); + adreno_gpu->info->name, gpu_config); if (ret) return ret; + pm_runtime_set_autosuspend_delay(&pdev->dev, DRM_MSM_INACTIVE_PERIOD); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_enable(&pdev->dev); + ret = request_firmware(&adreno_gpu->pm4, adreno_gpu->info->pm4fw, drm->dev); if (ret) { dev_err(drm->dev, "failed to load %s PM4 firmware: %d\n", @@ -373,55 +532,150 @@ } ret = request_firmware(&adreno_gpu->pfp, adreno_gpu->info->pfpfw, drm->dev); - if (ret) { + if (ret) dev_err(drm->dev, "failed to load %s PFP firmware: %d\n", adreno_gpu->info->pfpfw, ret); + return ret; } - mmu = gpu->mmu; - if (mmu) { - ret = mmu->funcs->attach(mmu, iommu_ports, - ARRAY_SIZE(iommu_ports)); - if (ret) - return ret; +void adreno_gpu_cleanup(struct adreno_gpu *adreno_gpu) +{ + struct msm_gpu *gpu = &adreno_gpu->base; + struct drm_device *dev = gpu->dev; + struct msm_drm_private *priv = dev->dev_private; + struct platform_device *pdev = priv->gpu_pdev; + + release_firmware(adreno_gpu->pm4); + release_firmware(adreno_gpu->pfp); + + pm_runtime_disable(&pdev->dev); + msm_gpu_cleanup(gpu); } - mutex_lock(&drm->struct_mutex); - adreno_gpu->memptrs_bo = msm_gem_new(drm, sizeof(*adreno_gpu->memptrs), - MSM_BO_UNCACHED); - mutex_unlock(&drm->struct_mutex); - if (IS_ERR(adreno_gpu->memptrs_bo)) { - ret = PTR_ERR(adreno_gpu->memptrs_bo); - adreno_gpu->memptrs_bo = NULL; - dev_err(drm->dev, "could not allocate memptrs: %d\n", ret); - return ret; +static void adreno_snapshot_os(struct msm_gpu *gpu, + struct msm_snapshot *snapshot) +{ + struct msm_snapshot_linux header; + + memset(&header, 0, sizeof(header)); + + header.osid = SNAPSHOT_OS_LINUX_V3; + strlcpy(header.release, utsname()->release, sizeof(header.release)); + strlcpy(header.version, utsname()->version, sizeof(header.version)); + + header.seconds = get_seconds(); + header.ctxtcount = 0; + + SNAPSHOT_HEADER(snapshot, header, SNAPSHOT_SECTION_OS, 0); } - adreno_gpu->memptrs = msm_gem_vaddr(adreno_gpu->memptrs_bo); - if (!adreno_gpu->memptrs) { - dev_err(drm->dev, "could not vmap memptrs\n"); - return -ENOMEM; +static void adreno_snapshot_ringbuffer(struct msm_gpu *gpu, + struct msm_snapshot *snapshot, struct msm_ringbuffer *ring) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct msm_snapshot_ringbuffer header; + unsigned int i, end = 0; + unsigned int *data = ring->start; + + memset(&header, 0, sizeof(header)); + + /* + * We only want to copy the active contents of each ring, so find the + * last valid entry in the ringbuffer + */ + for (i = 0; i < MSM_GPU_RINGBUFFER_SZ >> 2; i++) { + if (data[i]) + end = i; } - ret = msm_gem_get_iova(adreno_gpu->memptrs_bo, gpu->id, - &adreno_gpu->memptrs_iova); - if (ret) { - dev_err(drm->dev, "could not map memptrs: %d\n", ret); - return ret; + /* The dump always starts at 0 */ + header.start = 0; + header.end = end; + + /* This is the number of dwords being dumped */ + header.count = end + 1; + + /* This is the size of the actual ringbuffer */ + header.rbsize = MSM_GPU_RINGBUFFER_SZ >> 2; + + header.id = ring->id; + header.gpuaddr = ring->iova; + header.rptr = get_rptr(adreno_gpu, ring); + header.wptr = get_wptr(ring); + header.timestamp_queued = adreno_submitted_fence(gpu, ring); + header.timestamp_retired = ring->memptrs->fence; + + /* Write the header even if the ringbuffer data is empty */ + if (!SNAPSHOT_HEADER(snapshot, header, SNAPSHOT_SECTION_RB_V2, + header.count)) + return; + + SNAPSHOT_MEMCPY(snapshot, ring->start, header.count * sizeof(u32)); +} + +static void adreno_snapshot_ringbuffers(struct msm_gpu *gpu, + struct msm_snapshot *snapshot) +{ + struct msm_ringbuffer *ring; + int i; + + /* Write a new section for each ringbuffer */ + FOR_EACH_RING(gpu, ring, i) + adreno_snapshot_ringbuffer(gpu, snapshot, ring); +} + +void adreno_snapshot(struct msm_gpu *gpu, struct msm_snapshot *snapshot) +{ + adreno_snapshot_os(gpu, snapshot); + adreno_snapshot_ringbuffers(gpu, snapshot); +} + +/* Return the group struct associated with the counter id */ + +static struct adreno_counter_group *get_counter_group(struct msm_gpu *gpu, + u32 groupid) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + + if (!adreno_gpu->counter_groups) + return ERR_PTR(-ENODEV); + + if (groupid >= adreno_gpu->nr_counter_groups) + return ERR_PTR(-ENODEV); + + return (struct adreno_counter_group *) + adreno_gpu->counter_groups[groupid]; } +int adreno_get_counter(struct msm_gpu *gpu, u32 groupid, u32 countable, + u32 *lo, u32 *hi) +{ + struct adreno_counter_group *group = + get_counter_group(gpu, groupid); + + if (!IS_ERR_OR_NULL(group) && group->funcs.get) + return group->funcs.get(gpu, group, countable, lo, hi); + + return -ENODEV; +} + +u64 adreno_read_counter(struct msm_gpu *gpu, u32 groupid, int counterid) +{ + struct adreno_counter_group *group = + get_counter_group(gpu, groupid); + + if (!IS_ERR_OR_NULL(group) && group->funcs.read) + return group->funcs.read(gpu, group, counterid); + return 0; } -void adreno_gpu_cleanup(struct adreno_gpu *gpu) +void adreno_put_counter(struct msm_gpu *gpu, u32 groupid, int counterid) { - if (gpu->memptrs_bo) { - if (gpu->memptrs_iova) - msm_gem_put_iova(gpu->memptrs_bo, gpu->base.id); - drm_gem_object_unreference_unlocked(gpu->memptrs_bo); - } - release_firmware(gpu->pm4); - release_firmware(gpu->pfp); - msm_gpu_cleanup(&gpu->base); + struct adreno_counter_group *group = + get_counter_group(gpu, groupid); + + if (!IS_ERR_OR_NULL(group) && group->funcs.put) + group->funcs.put(gpu, group, counterid); } diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_gpu.h linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_gpu.h --- linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_gpu.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_gpu.h 2019-01-22 16:16:23.487246262 +0100 @@ -2,7 +2,7 @@ * Copyright (C) 2013 Red Hat * Author: Rob Clark * - * Copyright (c) 2014 The Linux Foundation. All rights reserved. + * Copyright (c) 2014,2016-2017 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published by @@ -24,10 +24,17 @@ #include "msm_gpu.h" +/* arrg, somehow fb.h is getting pulled in: */ +#undef ROP_COPY +#undef ROP_XOR + #include "adreno_common.xml.h" #include "adreno_pm4.xml.h" #define REG_ADRENO_DEFINE(_offset, _reg) [_offset] = (_reg) + 1 +#define REG_SKIP ~0 +#define REG_ADRENO_SKIP(_offset) [_offset] = REG_SKIP + /** * adreno_regs: List of registers that are used in across all * 3D devices. Each device type has different offset value for the same @@ -35,73 +42,21 @@ * and are indexed by the enumeration values defined in this enum */ enum adreno_regs { - REG_ADRENO_CP_DEBUG, - REG_ADRENO_CP_ME_RAM_WADDR, - REG_ADRENO_CP_ME_RAM_DATA, - REG_ADRENO_CP_PFP_UCODE_DATA, - REG_ADRENO_CP_PFP_UCODE_ADDR, - REG_ADRENO_CP_WFI_PEND_CTR, REG_ADRENO_CP_RB_BASE, + REG_ADRENO_CP_RB_BASE_HI, REG_ADRENO_CP_RB_RPTR_ADDR, + REG_ADRENO_CP_RB_RPTR_ADDR_HI, REG_ADRENO_CP_RB_RPTR, REG_ADRENO_CP_RB_WPTR, - REG_ADRENO_CP_PROTECT_CTRL, - REG_ADRENO_CP_ME_CNTL, REG_ADRENO_CP_RB_CNTL, - REG_ADRENO_CP_IB1_BASE, - REG_ADRENO_CP_IB1_BUFSZ, - REG_ADRENO_CP_IB2_BASE, - REG_ADRENO_CP_IB2_BUFSZ, - REG_ADRENO_CP_TIMESTAMP, - REG_ADRENO_CP_ME_RAM_RADDR, - REG_ADRENO_CP_ROQ_ADDR, - REG_ADRENO_CP_ROQ_DATA, - REG_ADRENO_CP_MERCIU_ADDR, - REG_ADRENO_CP_MERCIU_DATA, - REG_ADRENO_CP_MERCIU_DATA2, - REG_ADRENO_CP_MEQ_ADDR, - REG_ADRENO_CP_MEQ_DATA, - REG_ADRENO_CP_HW_FAULT, - REG_ADRENO_CP_PROTECT_STATUS, - REG_ADRENO_SCRATCH_ADDR, - REG_ADRENO_SCRATCH_UMSK, - REG_ADRENO_SCRATCH_REG2, - REG_ADRENO_RBBM_STATUS, - REG_ADRENO_RBBM_PERFCTR_CTL, - REG_ADRENO_RBBM_PERFCTR_LOAD_CMD0, - REG_ADRENO_RBBM_PERFCTR_LOAD_CMD1, - REG_ADRENO_RBBM_PERFCTR_LOAD_CMD2, - REG_ADRENO_RBBM_PERFCTR_PWR_1_LO, - REG_ADRENO_RBBM_INT_0_MASK, - REG_ADRENO_RBBM_INT_0_STATUS, - REG_ADRENO_RBBM_AHB_ERROR_STATUS, - REG_ADRENO_RBBM_PM_OVERRIDE2, - REG_ADRENO_RBBM_AHB_CMD, - REG_ADRENO_RBBM_INT_CLEAR_CMD, - REG_ADRENO_RBBM_SW_RESET_CMD, - REG_ADRENO_RBBM_CLOCK_CTL, - REG_ADRENO_RBBM_AHB_ME_SPLIT_STATUS, - REG_ADRENO_RBBM_AHB_PFP_SPLIT_STATUS, - REG_ADRENO_VPC_DEBUG_RAM_SEL, - REG_ADRENO_VPC_DEBUG_RAM_READ, - REG_ADRENO_VSC_SIZE_ADDRESS, - REG_ADRENO_VFD_CONTROL_0, - REG_ADRENO_VFD_INDEX_MAX, - REG_ADRENO_SP_VS_PVT_MEM_ADDR_REG, - REG_ADRENO_SP_FS_PVT_MEM_ADDR_REG, - REG_ADRENO_SP_VS_OBJ_START_REG, - REG_ADRENO_SP_FS_OBJ_START_REG, - REG_ADRENO_PA_SC_AA_CONFIG, - REG_ADRENO_SQ_GPR_MANAGEMENT, - REG_ADRENO_SQ_INST_STORE_MANAGMENT, - REG_ADRENO_TP0_CHICKEN, - REG_ADRENO_RBBM_RBBM_CTL, - REG_ADRENO_UCHE_INVALIDATE0, - REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_LO, - REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_HI, REG_ADRENO_REGISTER_MAX, }; +enum adreno_quirks { + ADRENO_QUIRK_TWO_PASS_USE_WFI = 1, + ADRENO_QUIRK_FAULT_DETECT_MASK = 2, +}; + struct adreno_rev { uint8_t core; uint8_t major; @@ -114,6 +69,7 @@ struct adreno_gpu_funcs { struct msm_gpu_funcs base; + int (*get_timestamp)(struct msm_gpu *gpu, uint64_t *value); }; struct adreno_info { @@ -127,10 +83,34 @@ const struct adreno_info *adreno_info(struct adreno_rev rev); -struct adreno_rbmemptrs { - volatile uint32_t rptr; - volatile uint32_t wptr; - volatile uint32_t fence; +struct adreno_counter { + u32 lo; + u32 hi; + u32 sel; + int load_bit; + u32 countable; + u32 refcount; + u64 value; +}; + +struct adreno_counter_group { + struct adreno_counter *counters; + size_t nr_counters; + spinlock_t lock; + struct { + int (*get)(struct msm_gpu *, + struct adreno_counter_group *, u32, u32 *, u32 *); + void (*enable)(struct msm_gpu *, + struct adreno_counter_group *, int, bool); + u64 (*read)(struct msm_gpu *, + struct adreno_counter_group *, int); + void (*put)(struct msm_gpu *, + struct adreno_counter_group *, int); + void (*save)(struct msm_gpu *, + struct adreno_counter_group *); + void (*restore)(struct msm_gpu *, + struct adreno_counter_group *); + } funcs; }; struct adreno_gpu { @@ -147,29 +127,24 @@ /* firmware: */ const struct firmware *pm4, *pfp; - /* ringbuffer rptr/wptr: */ - // TODO should this be in msm_ringbuffer? I think it would be - // different for z180.. - struct adreno_rbmemptrs *memptrs; - struct drm_gem_object *memptrs_bo; - uint32_t memptrs_iova; - /* * Register offsets are different between some GPUs. * GPU specific offsets will be exported by GPU specific * code (a3xx_gpu.c) and stored in this common location. */ const unsigned int *reg_offsets; + + uint32_t quirks; + uint32_t speed_bin; + + const struct adreno_counter_group **counter_groups; + int nr_counter_groups; }; #define to_adreno_gpu(x) container_of(x, struct adreno_gpu, base) /* platform config data (ie. from DT, or pdata) */ struct adreno_platform_config { struct adreno_rev rev; - uint32_t fast_rate, slow_rate, bus_freq; -#ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING - struct msm_bus_scale_pdata *bus_scale_table; -#endif }; #define ADRENO_IDLE_TIMEOUT msecs_to_jiffies(1000) @@ -186,6 +161,9 @@ __ret; \ }) +#define GPU_OF_NODE(_g) \ + (((struct msm_drm_private *) \ + ((_g)->dev->dev_private))->gpu_pdev->dev.of_node) static inline bool adreno_is_a3xx(struct adreno_gpu *gpu) { @@ -228,32 +206,55 @@ return gpu->revn == 420; } +static inline int adreno_is_a430(struct adreno_gpu *gpu) +{ + return gpu->revn == 430; +} + +static inline int adreno_is_a530(struct adreno_gpu *gpu) +{ + return gpu->revn == 530; +} + +static inline int adreno_is_a540(struct adreno_gpu *gpu) +{ + return gpu->revn == 540; +} + int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value); int adreno_hw_init(struct msm_gpu *gpu); -uint32_t adreno_last_fence(struct msm_gpu *gpu); +uint32_t adreno_submitted_fence(struct msm_gpu *gpu, + struct msm_ringbuffer *ring); void adreno_recover(struct msm_gpu *gpu); -int adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, - struct msm_file_private *ctx); -void adreno_flush(struct msm_gpu *gpu); -void adreno_idle(struct msm_gpu *gpu); +void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit); +void adreno_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring); +bool adreno_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring); #ifdef CONFIG_DEBUG_FS void adreno_show(struct msm_gpu *gpu, struct seq_file *m); #endif void adreno_dump_info(struct msm_gpu *gpu); void adreno_dump(struct msm_gpu *gpu); -void adreno_wait_ring(struct msm_gpu *gpu, uint32_t ndwords); +void adreno_wait_ring(struct msm_ringbuffer *ring, uint32_t ndwords); +struct msm_ringbuffer *adreno_active_ring(struct msm_gpu *gpu); int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev, - struct adreno_gpu *gpu, const struct adreno_gpu_funcs *funcs); + struct adreno_gpu *gpu, const struct adreno_gpu_funcs *funcs, + struct msm_gpu_config *config); void adreno_gpu_cleanup(struct adreno_gpu *gpu); +void adreno_snapshot(struct msm_gpu *gpu, struct msm_snapshot *snapshot); + +int adreno_get_counter(struct msm_gpu *gpu, u32 groupid, u32 countable, + u32 *lo, u32 *hi); +u64 adreno_read_counter(struct msm_gpu *gpu, u32 groupid, int counterid); +void adreno_put_counter(struct msm_gpu *gpu, u32 groupid, int counterid); /* ringbuffer helpers (the parts that are adreno specific) */ static inline void OUT_PKT0(struct msm_ringbuffer *ring, uint16_t regindx, uint16_t cnt) { - adreno_wait_ring(ring->gpu, cnt+1); + adreno_wait_ring(ring, cnt+1); OUT_RING(ring, CP_TYPE0_PKT | ((cnt-1) << 16) | (regindx & 0x7FFF)); } @@ -261,19 +262,49 @@ static inline void OUT_PKT2(struct msm_ringbuffer *ring) { - adreno_wait_ring(ring->gpu, 1); + adreno_wait_ring(ring, 1); OUT_RING(ring, CP_TYPE2_PKT); } static inline void OUT_PKT3(struct msm_ringbuffer *ring, uint8_t opcode, uint16_t cnt) { - adreno_wait_ring(ring->gpu, cnt+1); + adreno_wait_ring(ring, cnt+1); OUT_RING(ring, CP_TYPE3_PKT | ((cnt-1) << 16) | ((opcode & 0xFF) << 8)); } +static inline u32 PM4_PARITY(u32 val) +{ + return (0x9669 >> (0xF & (val ^ + (val >> 4) ^ (val >> 8) ^ (val >> 12) ^ + (val >> 16) ^ ((val) >> 20) ^ (val >> 24) ^ + (val >> 28)))) & 1; +} + +/* Maximum number of values that can be executed for one opcode */ +#define TYPE4_MAX_PAYLOAD 127 + +#define PKT4(_reg, _cnt) \ + (CP_TYPE4_PKT | ((_cnt) << 0) | (PM4_PARITY((_cnt)) << 7) | \ + (((_reg) & 0x3FFFF) << 8) | (PM4_PARITY((_reg)) << 27)) + +static inline void +OUT_PKT4(struct msm_ringbuffer *ring, uint16_t regindx, uint16_t cnt) +{ + adreno_wait_ring(ring, cnt + 1); + OUT_RING(ring, PKT4(regindx, cnt)); +} + +static inline void +OUT_PKT7(struct msm_ringbuffer *ring, uint8_t opcode, uint16_t cnt) +{ + adreno_wait_ring(ring, cnt + 1); + OUT_RING(ring, CP_TYPE7_PKT | (cnt << 0) | (PM4_PARITY(cnt) << 15) | + ((opcode & 0x7F) << 16) | (PM4_PARITY(opcode) << 23)); +} + /* - * adreno_checkreg_off() - Checks the validity of a register enum + * adreno_reg_check() - Checks the validity of a register enum * @gpu: Pointer to struct adreno_gpu * @offset_name: The register enum that is checked */ @@ -284,6 +315,16 @@ !gpu->reg_offsets[offset_name]) { BUG(); } + + /* + * REG_SKIP is a special value that tell us that the register in + * question isn't implemented on target but don't trigger a BUG(). This + * is used to cleanly implement adreno_gpu_write64() and + * adreno_gpu_read64() in a generic fashion + */ + if (gpu->reg_offsets[offset_name] == REG_SKIP) + return false; + return true; } @@ -305,4 +346,40 @@ gpu_write(&gpu->base, reg - 1, data); } +static inline void adreno_gpu_write64(struct adreno_gpu *gpu, + enum adreno_regs lo, enum adreno_regs hi, u64 data) +{ + adreno_gpu_write(gpu, lo, lower_32_bits(data)); + adreno_gpu_write(gpu, hi, upper_32_bits(data)); +} + +static inline uint32_t get_wptr(struct msm_ringbuffer *ring) +{ + return (ring->cur - ring->start) % (MSM_GPU_RINGBUFFER_SZ >> 2); +} + +/* + * Given a register and a count, return a value to program into + * REG_CP_PROTECT_REG(n) - this will block both reads and writes for _len + * registers starting at _reg. + * + * The register base needs to be a multiple of the length. If it is not, the + * hardware will quietly mask off the bits for you and shift the size. For + * example, if you intend the protection to start at 0x07 for a length of 4 + * (0x07-0x0A) the hardware will actually protect (0x04-0x07) which might + * expose registers you intended to protect! + */ +#define ADRENO_PROTECT_RW(_reg, _len) \ + ((1 << 30) | (1 << 29) | \ + ((ilog2((_len)) & 0x1F) << 24) | (((_reg) << 2) & 0xFFFFF)) + +/* + * Same as above, but allow reads over the range. For areas of mixed use (such + * as performance counters) this allows us to protect a much larger range with a + * single register + */ +#define ADRENO_PROTECT_RDONLY(_reg, _len) \ + ((1 << 29) \ + ((ilog2((_len)) & 0x1F) << 24) | (((_reg) << 2) & 0xFFFFF)) + #endif /* __ADRENO_GPU_H__ */ diff -ruw linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h --- linux-4.4.115/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h 2019-01-22 16:16:23.487246262 +0100 @@ -8,17 +8,19 @@ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 398 bytes, from 2015-09-24 17:25:31) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 32901 bytes, from 2015-05-20 20:03:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 10755 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 14968 bytes, from 2015-05-20 20:12:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 67771 bytes, from 2015-09-14 20:46:55) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 63970 bytes, from 2015-09-14 20:50:12) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2015-09-24 17:30:00) +- ./adreno.xml ( 431 bytes, from 2016-10-24 21:12:27) +- ./freedreno_copyright.xml ( 1572 bytes, from 2016-10-24 21:12:27) +- ./adreno/a2xx.xml ( 32901 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_common.xml ( 12025 bytes, from 2016-10-24 21:12:27) +- ./adreno/adreno_pm4.xml ( 19684 bytes, from 2016-10-24 21:12:27) +- ./adreno/a3xx.xml ( 83840 bytes, from 2016-10-24 21:12:27) +- ./adreno/a4xx.xml ( 110708 bytes, from 2016-10-24 21:12:27) +- ./adreno/a5xx.xml ( 81207 bytes, from 2016-10-26 19:36:59) +- ./adreno/ocmem.xml ( 1773 bytes, from 2016-10-24 21:12:27) -Copyright (C) 2013-2015 by the following authors: +Copyright (C) 2013-2016 by the following authors: - Rob Clark (robclark) +- Ilia Mirkin (imirkin) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -57,6 +59,7 @@ RST_PIX_CNT = 13, RST_VTX_CNT = 14, TILE_FLUSH = 15, + STAT_EVENT = 16, CACHE_FLUSH_AND_INV_TS_EVENT = 20, ZPASS_DONE = 21, CACHE_FLUSH_AND_INV_EVENT = 22, @@ -81,7 +84,6 @@ DI_PT_LINESTRIP_ADJ = 11, DI_PT_TRI_ADJ = 12, DI_PT_TRISTRIP_ADJ = 13, - DI_PT_PATCHES = 34, }; enum pc_di_src_sel { @@ -109,11 +111,15 @@ CP_TYPE1_PKT = 0x40000000, CP_TYPE2_PKT = 0x80000000, CP_TYPE3_PKT = 0xc0000000, + CP_TYPE4_PKT = 0x40000000, + CP_TYPE7_PKT = 0x70000000, }; enum adreno_pm4_type3_packets { CP_ME_INIT = 72, CP_NOP = 16, + CP_PREEMPT_ENABLE = 28, + CP_PREEMPT_TOKEN = 30, CP_INDIRECT_BUFFER = 63, CP_INDIRECT_BUFFER_PFD = 55, CP_WAIT_FOR_IDLE = 38, @@ -162,6 +168,7 @@ CP_TEST_TWO_MEMS = 113, CP_REG_WR_NO_CTXT = 120, CP_RECORD_PFP_TIMESTAMP = 17, + CP_SET_SECURE_MODE = 102, CP_WAIT_FOR_ME = 19, CP_SET_DRAW_STATE = 67, CP_DRAW_INDX_OFFSET = 56, @@ -172,6 +179,26 @@ CP_UNKNOWN_1A = 26, CP_UNKNOWN_4E = 78, CP_WIDE_REG_WRITE = 116, + CP_SCRATCH_TO_REG = 77, + CP_REG_TO_SCRATCH = 74, + CP_WAIT_MEM_WRITES = 18, + CP_COND_REG_EXEC = 71, + CP_MEM_TO_REG = 66, + CP_EXEC_CS = 51, + CP_PERFCOUNTER_ACTION = 80, + CP_SMMU_TABLE_UPDATE = 83, + CP_CONTEXT_REG_BUNCH = 92, + CP_YIELD_ENABLE = 28, + CP_SKIP_IB2_ENABLE_GLOBAL = 29, + CP_SKIP_IB2_ENABLE_LOCAL = 35, + CP_SET_SUBDRAW_SIZE = 53, + CP_SET_VISIBILITY_OVERRIDE = 100, + CP_PREEMPT_ENABLE_GLOBAL = 105, + CP_PREEMPT_ENABLE_LOCAL = 106, + CP_CONTEXT_SWITCH_YIELD = 107, + CP_SET_RENDER_MODE = 108, + CP_COMPUTE_CHECKPOINT = 110, + CP_MEM_TO_MEM = 115, IN_IB_PREFETCH_END = 23, IN_SUBBLK_PREFETCH = 31, IN_INSTR_PREFETCH = 32, @@ -190,6 +217,7 @@ SB_VERT_SHADER = 4, SB_GEOM_SHADER = 5, SB_FRAG_SHADER = 6, + SB_COMPUTE_SHADER = 7, }; enum adreno_state_type { @@ -199,7 +227,11 @@ enum adreno_state_src { SS_DIRECT = 0, + SS_INVALID_ALL_IC = 2, + SS_INVALID_PART_IC = 3, SS_INDIRECT = 4, + SS_INDIRECT_TCM = 5, + SS_INDIRECT_STM = 6, }; enum a4xx_index_size { @@ -227,7 +259,7 @@ { return ((val) << CP_LOAD_STATE_0_STATE_BLOCK__SHIFT) & CP_LOAD_STATE_0_STATE_BLOCK__MASK; } -#define CP_LOAD_STATE_0_NUM_UNIT__MASK 0x7fc00000 +#define CP_LOAD_STATE_0_NUM_UNIT__MASK 0xffc00000 #define CP_LOAD_STATE_0_NUM_UNIT__SHIFT 22 static inline uint32_t CP_LOAD_STATE_0_NUM_UNIT(uint32_t val) { @@ -379,7 +411,12 @@ { return ((val) << CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__SHIFT) & CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__MASK; } -#define CP_DRAW_INDX_OFFSET_0_TESSELLATE 0x00000100 +#define CP_DRAW_INDX_OFFSET_0_VIS_CULL__MASK 0x00000300 +#define CP_DRAW_INDX_OFFSET_0_VIS_CULL__SHIFT 8 +static inline uint32_t CP_DRAW_INDX_OFFSET_0_VIS_CULL(enum pc_di_vis_cull_mode val) +{ + return ((val) << CP_DRAW_INDX_OFFSET_0_VIS_CULL__SHIFT) & CP_DRAW_INDX_OFFSET_0_VIS_CULL__MASK; +} #define CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__MASK 0x00000c00 #define CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__SHIFT 10 static inline uint32_t CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(enum a4xx_index_size val) @@ -499,5 +536,102 @@ return ((val) << CP_SET_BIN_DATA_1_BIN_SIZE_ADDRESS__SHIFT) & CP_SET_BIN_DATA_1_BIN_SIZE_ADDRESS__MASK; } +#define REG_CP_REG_TO_MEM_0 0x00000000 +#define CP_REG_TO_MEM_0_REG__MASK 0x0000ffff +#define CP_REG_TO_MEM_0_REG__SHIFT 0 +static inline uint32_t CP_REG_TO_MEM_0_REG(uint32_t val) +{ + return ((val) << CP_REG_TO_MEM_0_REG__SHIFT) & CP_REG_TO_MEM_0_REG__MASK; +} +#define CP_REG_TO_MEM_0_CNT__MASK 0x3ff80000 +#define CP_REG_TO_MEM_0_CNT__SHIFT 19 +static inline uint32_t CP_REG_TO_MEM_0_CNT(uint32_t val) +{ + return ((val) << CP_REG_TO_MEM_0_CNT__SHIFT) & CP_REG_TO_MEM_0_CNT__MASK; +} +#define CP_REG_TO_MEM_0_64B 0x40000000 +#define CP_REG_TO_MEM_0_ACCUMULATE 0x80000000 + +#define REG_CP_REG_TO_MEM_1 0x00000001 +#define CP_REG_TO_MEM_1_DEST__MASK 0xffffffff +#define CP_REG_TO_MEM_1_DEST__SHIFT 0 +static inline uint32_t CP_REG_TO_MEM_1_DEST(uint32_t val) +{ + return ((val) << CP_REG_TO_MEM_1_DEST__SHIFT) & CP_REG_TO_MEM_1_DEST__MASK; +} + +#define REG_CP_DISPATCH_COMPUTE_0 0x00000000 + +#define REG_CP_DISPATCH_COMPUTE_1 0x00000001 +#define CP_DISPATCH_COMPUTE_1_X__MASK 0xffffffff +#define CP_DISPATCH_COMPUTE_1_X__SHIFT 0 +static inline uint32_t CP_DISPATCH_COMPUTE_1_X(uint32_t val) +{ + return ((val) << CP_DISPATCH_COMPUTE_1_X__SHIFT) & CP_DISPATCH_COMPUTE_1_X__MASK; +} + +#define REG_CP_DISPATCH_COMPUTE_2 0x00000002 +#define CP_DISPATCH_COMPUTE_2_Y__MASK 0xffffffff +#define CP_DISPATCH_COMPUTE_2_Y__SHIFT 0 +static inline uint32_t CP_DISPATCH_COMPUTE_2_Y(uint32_t val) +{ + return ((val) << CP_DISPATCH_COMPUTE_2_Y__SHIFT) & CP_DISPATCH_COMPUTE_2_Y__MASK; +} + +#define REG_CP_DISPATCH_COMPUTE_3 0x00000003 +#define CP_DISPATCH_COMPUTE_3_Z__MASK 0xffffffff +#define CP_DISPATCH_COMPUTE_3_Z__SHIFT 0 +static inline uint32_t CP_DISPATCH_COMPUTE_3_Z(uint32_t val) +{ + return ((val) << CP_DISPATCH_COMPUTE_3_Z__SHIFT) & CP_DISPATCH_COMPUTE_3_Z__MASK; +} + +#define REG_CP_SET_RENDER_MODE_0 0x00000000 + +#define REG_CP_SET_RENDER_MODE_1 0x00000001 +#define CP_SET_RENDER_MODE_1_ADDR_0_LO__MASK 0xffffffff +#define CP_SET_RENDER_MODE_1_ADDR_0_LO__SHIFT 0 +static inline uint32_t CP_SET_RENDER_MODE_1_ADDR_0_LO(uint32_t val) +{ + return ((val) << CP_SET_RENDER_MODE_1_ADDR_0_LO__SHIFT) & CP_SET_RENDER_MODE_1_ADDR_0_LO__MASK; +} + +#define REG_CP_SET_RENDER_MODE_2 0x00000002 +#define CP_SET_RENDER_MODE_2_ADDR_0_HI__MASK 0xffffffff +#define CP_SET_RENDER_MODE_2_ADDR_0_HI__SHIFT 0 +static inline uint32_t CP_SET_RENDER_MODE_2_ADDR_0_HI(uint32_t val) +{ + return ((val) << CP_SET_RENDER_MODE_2_ADDR_0_HI__SHIFT) & CP_SET_RENDER_MODE_2_ADDR_0_HI__MASK; +} + +#define REG_CP_SET_RENDER_MODE_3 0x00000003 +#define CP_SET_RENDER_MODE_3_GMEM_ENABLE 0x00000010 + +#define REG_CP_SET_RENDER_MODE_4 0x00000004 + +#define REG_CP_SET_RENDER_MODE_5 0x00000005 +#define CP_SET_RENDER_MODE_5_ADDR_1_LEN__MASK 0xffffffff +#define CP_SET_RENDER_MODE_5_ADDR_1_LEN__SHIFT 0 +static inline uint32_t CP_SET_RENDER_MODE_5_ADDR_1_LEN(uint32_t val) +{ + return ((val) << CP_SET_RENDER_MODE_5_ADDR_1_LEN__SHIFT) & CP_SET_RENDER_MODE_5_ADDR_1_LEN__MASK; +} + +#define REG_CP_SET_RENDER_MODE_6 0x00000006 +#define CP_SET_RENDER_MODE_6_ADDR_1_LO__MASK 0xffffffff +#define CP_SET_RENDER_MODE_6_ADDR_1_LO__SHIFT 0 +static inline uint32_t CP_SET_RENDER_MODE_6_ADDR_1_LO(uint32_t val) +{ + return ((val) << CP_SET_RENDER_MODE_6_ADDR_1_LO__SHIFT) & CP_SET_RENDER_MODE_6_ADDR_1_LO__MASK; +} + +#define REG_CP_SET_RENDER_MODE_7 0x00000007 +#define CP_SET_RENDER_MODE_7_ADDR_1_HI__MASK 0xffffffff +#define CP_SET_RENDER_MODE_7_ADDR_1_HI__SHIFT 0 +static inline uint32_t CP_SET_RENDER_MODE_7_ADDR_1_HI(uint32_t val) +{ + return ((val) << CP_SET_RENDER_MODE_7_ADDR_1_HI__SHIFT) & CP_SET_RENDER_MODE_7_ADDR_1_HI__MASK; +} + #endif /* ADRENO_PM4_XML */ diff -ruw linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi.h linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi.h --- linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi.h 2019-01-22 16:16:23.491246298 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * Copyright (c) 2015,2017, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -89,7 +89,7 @@ u32 *clk_pre, u32 *clk_post); void msm_dsi_manager_phy_disable(int id); int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg); -bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 iova, u32 len); +bool msm_dsi_manager_cmd_xfer_trigger(int id, u64 iova, u32 len); int msm_dsi_manager_register(struct msm_dsi *msm_dsi); void msm_dsi_manager_unregister(struct msm_dsi *msm_dsi); @@ -143,7 +143,7 @@ int msm_dsi_host_cmd_rx(struct mipi_dsi_host *host, const struct mipi_dsi_msg *msg); void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host, - u32 iova, u32 len); + u64 iova, u32 len); int msm_dsi_host_enable(struct mipi_dsi_host *host); int msm_dsi_host_disable(struct mipi_dsi_host *host); int msm_dsi_host_power_on(struct mipi_dsi_host *host); diff -ruw linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi_host.c linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi_host.c --- linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi_host.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi_host.c 2019-10-29 09:26:23.625203002 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * Copyright (c) 2015,2017 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -836,24 +836,21 @@ { struct drm_device *dev = msm_host->dev; int ret; - u32 iova; + u64 iova; - mutex_lock(&dev->struct_mutex); msm_host->tx_gem_obj = msm_gem_new(dev, size, MSM_BO_UNCACHED); if (IS_ERR(msm_host->tx_gem_obj)) { ret = PTR_ERR(msm_host->tx_gem_obj); pr_err("%s: failed to allocate gem, %d\n", __func__, ret); msm_host->tx_gem_obj = NULL; - mutex_unlock(&dev->struct_mutex); return ret; } - ret = msm_gem_get_iova_locked(msm_host->tx_gem_obj, 0, &iova); + ret = msm_gem_get_iova(msm_host->tx_gem_obj, NULL, &iova); if (ret) { pr_err("%s: failed to get iova, %d\n", __func__, ret); return ret; } - mutex_unlock(&dev->struct_mutex); if (iova & 0x07) { pr_err("%s: buf NOT 8 bytes aligned\n", __func__); @@ -974,7 +971,7 @@ static int dsi_cmd_dma_tx(struct msm_dsi_host *msm_host, int len) { int ret; - u32 iova; + uint64_t iova; bool triggered; ret = msm_gem_get_iova(msm_host->tx_gem_obj, 0, &iova); @@ -1750,11 +1747,12 @@ return ret; } -void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host, u32 iova, u32 len) +void msm_dsi_host_cmd_xfer_commit(struct mipi_dsi_host *host, u64 iova, u32 len) { struct msm_dsi_host *msm_host = to_msm_dsi_host(host); - dsi_write(msm_host, REG_DSI_DMA_BASE, iova); + /* FIXME: Verify that the iova < 32 bits? */ + dsi_write(msm_host, REG_DSI_DMA_BASE, lower_32_bits(iova)); dsi_write(msm_host, REG_DSI_DMA_LEN, len); dsi_write(msm_host, REG_DSI_TRIG_DMA, 1); diff -ruw linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi_manager.c linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi_manager.c --- linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi_manager.c 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi_manager.c 2019-10-29 09:26:23.625203002 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * Copyright (c) 2015,2017 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -774,7 +774,7 @@ return ret; } -bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 iova, u32 len) +bool msm_dsi_manager_cmd_xfer_trigger(int id, u64 iova, u32 len) { struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct msm_dsi *msm_dsi0 = dsi_mgr_get_dsi(DSI_0); diff -ruw linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi.xml.h linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi.xml.h --- linux-4.4.115/drivers/gpu/drm/msm/dsi/dsi.xml.h 2018-02-03 17:04:31.000000000 +0100 +++ linux-4.4.115-fbx/drivers/gpu/drm/msm/dsi/dsi.xml.h 2019-01-22 16:16:23.491246298 +0100 @@ -9,7 +9,7 @@ The rules-ng-ng source files this header was generated from are: - /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2015-05-20 20:03:14) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1453 bytes, from 2015-05-20 20:03:07) +- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2016-02-10 17:07:21) - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2015-05-20 20:03:14) - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2015-09-18 12:07:28) - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37194 bytes, from 2015-09-18 12:07:28) @@ -17,11 +17,12 @@ - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2015-10-22 16:35:02) - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2015-05-20 20:03:14) - /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2015-05-20 20:03:07) -- /home/robc