igb: don't unmap hw_addr if its NULL

Started by Jarod Wilson on Sep 10, 2015 · 5 messages from 3 authors · Last reply Sep 10, 2015

Subsystems (3)

Maintainers (8)

Show 4 more
STALE4069d

open the first message on its own page

  1. [PATCH] igb: don't unmap hw_addr if its NULL
    Related patches
  2. [PATCH v2] igb: don't unmap NULL hw_addr
    Related patches

Jarod Wilson 

I've got a startech thunderbolt dock someone loaned me, which among other
things, has the following device in it:

08:00.0 Ethernet controller: Intel Corporation I210 Gigabit Network Connection (rev 03)

This hotplugs just fine (kernel 4.2.0 plus a patch or two here):

[  863.020315] igb: Intel(R) Gigabit Ethernet Network Driver - version 5.2.18-k
[  863.020316] igb: Copyright (c) 2007-2014 Intel Corporation.
[  863.028657] igb 0000:08:00.0: enabling device (0000 -> 0002)
[  863.062089] igb 0000:08:00.0: added PHC on eth0
[  863.062090] igb 0000:08:00.0: Intel(R) Gigabit Ethernet Network Connection
[  863.062091] igb 0000:08:00.0: eth0: (PCIe:2.5Gb/s:Width x1) e8:ea:6a:00:1b:2a
[  863.062194] igb 0000:08:00.0: eth0: PBA No: 000200-000
[  863.062196] igb 0000:08:00.0: Using MSI-X interrupts. 4 rx queue(s), 4 tx queue(s)
[  863.064889] igb 0000:08:00.0 enp8s0: renamed from eth0

But disconnecting it is another story:

[ 1002.807932] igb 0000:08:00.0: removed PHC on enp8s0
[ 1002.807944] igb 0000:08:00.0 enp8s0: PCIe link lost, device now detached
[ 1003.341141] ------------[ cut here ]------------
[ 1003.341148] WARNING: CPU: 0 PID: 199 at lib/iomap.c:43 bad_io_access+0x38/0x40()
[ 1003.341149] Bad IO access at port 0x0 ()
[ 1003.342767] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_rawmidi igb dca firewire_ohci firewire_core crc_itu_t rfcomm ctr ccm arc4 iwlmvm mac80211 fuse xt_CHECKSUM ipt_MASQUERADE
nf_nat_masquerade_ipv4 tun ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat
nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat
nf_conntrack iptable_mangle iptable_security iptable_raw iptable_filter bnep dm_mirror dm_region_hash dm_log dm_mod coretemp x86_pkg_temp_thermal intel_powerclamp kvm_intel snd_hda_codec_hdmi kvm
crct10dif_pclmul crc32_pclmul ghash_clmulni_intel drbg
[ 1003.342793]  ansi_cprng aesni_intel hp_wmi aes_x86_64 iTCO_wdt lrw iTCO_vendor_support ppdev gf128mul sparse_keymap glue_helper ablk_helper cryptd snd_hda_codec_realtek snd_hda_codec_generic
microcode snd_hda_intel uvcvideo iwlwifi snd_hda_codec videobuf2_vmalloc videobuf2_memops snd_hda_core videobuf2_core snd_hwdep btusb v4l2_common btrtl snd_seq btbcm btintel videodev cfg80211
snd_seq_device rtsx_pci_ms bluetooth pcspkr input_leds i2c_i801 media parport_pc memstick rfkill sg lpc_ich snd_pcm 8250_fintek parport joydev snd_timer snd soundcore hp_accel ie31200_edac
mei_me lis3lv02d edac_core input_polldev mei hp_wireless shpchp tpm_infineon sch_fq_codel nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables autofs4 xfs libcrc32c sd_mod sr_mod cdrom
rtsx_pci_sdmmc mmc_core crc32c_intel serio_raw rtsx_pci
[ 1003.342822]  nouveau ahci libahci mxm_wmi e1000e xhci_pci hwmon ptp drm_kms_helper pps_core xhci_hcd ttm wmi video ipv6
[ 1003.342839] CPU: 0 PID: 199 Comm: kworker/0:2 Not tainted 4.2.0-2.el7_UNSUPPORTED.x86_64 #1
[ 1003.342840] Hardware name: Hewlett-Packard HP ZBook 15 G2/2253, BIOS M70 Ver. 01.07 02/26/2015
[ 1003.342843] Workqueue: pciehp-3 pciehp_power_thread
[ 1003.342844]  ffffffff81a90655 ffff8804866d3b48 ffffffff8164763a 0000000000000000
[ 1003.342846]  ffff8804866d3b98 ffff8804866d3b88 ffffffff8107134a ffff8804866d3b88

45 more lines

diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index e174fbb..a5e0022 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -2823,7 +2823,8 @@ static void igb_remove(struct pci_dev *pdev)
 
 	igb_clear_interrupt_scheme(adapter);
 
-	pci_iounmap(pdev, hw->hw_addr);
+	if (hw->hw_addr)
+		pci_iounmap(pdev, hw->hw_addr);
 	if (hw->flash_address)
 		iounmap(hw->flash_address);
 	pci_release_selected_regions(pdev,
-- 
1.8.3.1

RE: [Intel-wired-lan] [PATCH] igb: don't unmap hw_addr if its NULL

Fujinaka, Todd replied to Jarod Wilson 

Thanks! I tried testing this sort of thing last spring, but there just wasn't a lot of hardware available to me and things weren't breaking with my setup (not sure why).

ACK.

Todd Fujinaka
Software Application Engineer
Networking Division (ND)
Intel Corporation
todd.fujinaka@intel.com
(503) 712-4565

-----Original Message-----
From: Intel-wired-lan [mailto:<redacted>] On Behalf Of Jarod Wilson
Sent: Wednesday, September 09, 2015 9:07 PM
To: <redacted>
Cc: <redacted>; <redacted>; Jarod Wilson
Subject: [Intel-wired-lan] [PATCH] igb: don't unmap hw_addr if its NULL

I've got a startech thunderbolt dock someone loaned me, which among other things, has the following device in it:

08:00.0 Ethernet controller: Intel Corporation I210 Gigabit Network Connection (rev 03)

This hotplugs just fine (kernel 4.2.0 plus a patch or two here):

[  863.020315] igb: Intel(R) Gigabit Ethernet Network Driver - version 5.2.18-k [  863.020316] igb: Copyright (c) 2007-2014 Intel Corporation.
[  863.028657] igb 0000:08:00.0: enabling device (0000 -> 0002) [  863.062089] igb 0000:08:00.0: added PHC on eth0 [  863.062090] igb 0000:08:00.0: Intel(R) Gigabit Ethernet Network Connection [  863.062091] igb 0000:08:00.0: eth0: (PCIe:2.5Gb/s:Width x1) e8:ea:6a:00:1b:2a [  863.062194] igb 0000:08:00.0: eth0: PBA No: 000200-000 [  863.062196] igb 0000:08:00.0: Using MSI-X interrupts. 4 rx queue(s), 4 tx queue(s) [  863.064889] igb 0000:08:00.0 enp8s0: renamed from eth0

But disconnecting it is another story:

[ 1002.807932] igb 0000:08:00.0: removed PHC on enp8s0 [ 1002.807944] igb 0000:08:00.0 enp8s0: PCIe link lost, device now detached [ 1003.341141] ------------[ cut here ]------------ [ 1003.341148] WARNING: CPU: 0 PID: 199 at lib/iomap.c:43 bad_io_access+0x38/0x40() [ 1003.341149] Bad IO access at port 0x0 () [ 1003.342767] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_rawmidi igb dca firewire_ohci firewire_core crc_itu_t rfcomm ctr ccm arc4 iwlmvm mac80211 fuse xt_CHECKSUM ipt_MASQUERADE
nf_nat_masquerade_ipv4 tun ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat
nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw iptable_filter bnep dm_mirror dm_region_hash dm_log dm_mod coretemp x86_pkg_temp_thermal intel_powerclamp kvm_intel snd_hda_codec_hdmi kvm crct10dif_pclmul crc32_pclmul ghash_clmulni_intel drbg [ 1003.342793]  ansi_cprng aesni_intel hp_wmi aes_x86_64 iTCO_wdt lrw iTCO_vendor_support ppdev gf128mul sparse_keymap glue_helper ablk_helper cryptd snd_hda_codec_realtek snd_hda_codec_generic microcode snd_hda_intel uvcvideo iwlwifi snd_hda_codec videobuf2_vmalloc videobuf2_memops snd_hda_core videobuf2_core snd_hwdep btusb v4l2_common btrtl snd_seq btbcm btinte
 l videodev cfg80211 snd_seq_device rtsx_pci_ms bluetooth pcspkr input_leds i2c_i801 media parport_pc memstick rfkill sg lpc_ich snd_pcm 8250_fintek parport joydev snd_timer snd soundcore hp_accel ie31200_edac mei_me lis3lv02d edac_core input_polldev mei hp_wireless shpchp tpm_infineon sch_fq_codel nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables autofs4 xfs libcrc32c sd_mod sr_mod cdrom rtsx_pci_sdmmc mmc_core crc32c_intel serio_raw rtsx_pci [ 1003.342822]  nouveau ahci libahci mxm_wmi e1000e xhci_pci hwmon ptp drm_kms_helper pps_core xhci_hcd ttm wmi video ipv6 [ 1003.342839] CPU: 0 PID: 199 Comm: kworker/0:2 Not tainted 4.2.0-2.el7_UNSUPPORTED.x86_64 #1 [ 1003.342840] Hardware name: Hewlett-Packard HP ZBook 15 G2/2253, BIOS M70 Ver. 01.07 02/26/2015 [ 1003.342843] Workqueue: pcieh
 p-3 pciehp_power_thread [ 1003.342844]  ffffffff81a90655 ffff8804866d3b48 ffffffff8164763a 0000000000000000 [ 1003.342846]  ffff8804866d3b98 ffff8804866d3b88 ffffffff8107134a ffff8804866d3b88 [ 1003.342847]  ffff880486f46000 ffff88046c8a8000 ffff880486f46840 ffff88046c8a8098 [ 1003.342848] Call Trace:
[ 1003.342852]  [<ffffffff8164763a>] dump_stack+0x45/0x57 [ 1003.342855]  [<ffffffff8107134a>] warn_slowpath_common+0x8a/0xc0 [ 1003.342857]  [<ffffffff810713c6>] warn_slowpath_fmt+0x46/0x50 [ 1003.342859]  [<ffffffff8133719e>] ? pci_disable_msix+0x3e/0x50 [ 1003.342860]  [<ffffffff812f6328>] bad_io_access+0x38/0x40 [ 1003.342861]  [<ffffffff812f6567>] pci_iounmap+0x27/0x40 [ 1003.342865]  [<ffffffffa0b728d7>] igb_remove+0xc7/0x160 [igb] [ 1003.342867]  [<ffffffff8132189f>] pci_device_remove+0x3f/0xc0 [ 1003.342869]  [<ffffffff81433426>] __device_release_driver+0x96/0x130
[ 1003.342870]  [<ffffffff814334e3>] device_release_driver+0x23/0x30 [ 1003.342871]  [<ffffffff8131b404>] pci_stop_bus_device+0x94/0xa0 [ 1003.342872]  [<ffffffff8131b3ad>] pci_stop_bus_device+0x3d/0xa0 [ 1003.342873]  [<ffffffff8131b3ad>] pci_stop_bus_device+0x3d/0xa0 [ 1003.342874]  [<ffffffff8131b516>] pci_stop_and_remove_bus_device+0x16/0x30
[ 1003.342876]  [<ffffffff81333f5b>] pciehp_unconfigure_device+0x9b/0x180
[ 1003.342877]  [<ffffffff81333a73>] pciehp_disable_slot+0x43/0xb0 [ 1003.342878]  [<ffffffff81333b6d>] pciehp_power_thread+0x8d/0xb0 [ 1003.342885]  [<ffffffff810881b2>] process_one_work+0x152/0x3d0 [ 1003.342886]  [<ffffffff8108854a>] worker_thread+0x11a/0x460 [ 1003.342887]  [<ffffffff81088430>] ? process_one_work+0x3d0/0x3d0 [ 1003.342890]  [<ffffffff8108ddd9>] kthread+0xc9/0xe0 [ 1003.342891]  [<ffffffff8108dd10>] ? kthread_create_on_node+0x180/0x180
[ 1003.342893]  [<ffffffff8164e29f>] ret_from_fork+0x3f/0x70 [ 1003.342894]  [<ffffffff8108dd10>] ? kthread_create_on_node+0x180/0x180
[ 1003.342895] ---[ end trace 65a77e06d5aa9358 ]---

13 more lines

diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index e174fbb..a5e0022 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -2823,7 +2823,8 @@ static void igb_remove(struct pci_dev *pdev)
 
 	igb_clear_interrupt_scheme(adapter);
 
-	pci_iounmap(pdev, hw->hw_addr);
+	if (hw->hw_addr)
+		pci_iounmap(pdev, hw->hw_addr);
 	if (hw->flash_address)
 		iounmap(hw->flash_address);
 	pci_release_selected_regions(pdev,
--
1.8.3.1

_______________________________________________
Intel-wired-lan mailing list
Intel-wired-lan@lists.osuosl.org
http://lists.osuosl.org/mailman/listinfo/intel-wired-lan

Rustad, Mark D replied to Jarod Wilson

quoted hunk
On Sep 9, 2015, at 9:07 PM, Jarod Wilson [off-list ref] wrote:

diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index e174fbb..a5e0022 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -2823,7 +2823,8 @@ static void igb_remove(struct pci_dev *pdev)

	igb_clear_interrupt_scheme(adapter);

-	pci_iounmap(pdev, hw->hw_addr);
+	if (hw->hw_addr)
+		pci_iounmap(pdev, hw->hw_addr);
	if (hw->flash_address)
		iounmap(hw->flash_address);
	pci_release_selected_regions(pdev,
I don't think that this is entirely the right solution. In ixgbe we have a separate pointer, io_addr, used to manage the resource, so that the space can be freed even after hw_addr is cleared. With the approach above, the pci_iounmap will not ever be called on the space. You can see how ixgbe is doing it.

--
Mark Rustad, Networking Division, Intel Corporation

Jarod Wilson replied to Rustad, Mark D 

Rustad, Mark D wrote:
quoted
On Sep 9, 2015, at 9:07 PM, Jarod Wilson[off-list ref]  wrote:

diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index e174fbb..a5e0022 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -2823,7 +2823,8 @@ static void igb_remove(struct pci_dev *pdev)

	igb_clear_interrupt_scheme(adapter);

-	pci_iounmap(pdev, hw->hw_addr);
+	if (hw->hw_addr)
+		pci_iounmap(pdev, hw->hw_addr);
	if (hw->flash_address)
		iounmap(hw->flash_address);
	pci_release_selected_regions(pdev,
I don't think that this is entirely the right solution. In ixgbe we
have a separate pointer, io_addr, used to manage the resource, so that
the space can be freed even after hw_addr is cleared. With the approach
above, the pci_iounmap will not ever be called on the space. You can see
how ixgbe is doing it.
Ah, you're right, that had crossed my mind briefly as well. I've got a revised version here now that adds an io_addr to igb_adapter and uses that for unmap. I'll post it after I double-check that it builds and functions as expected.

-- 
Jarod Wilson
jarod@redhat.com

[PATCH v2] igb: don't unmap NULL hw_addr

Jarod Wilson replied to Rustad, Mark D 

I've got a startech thunderbolt dock someone loaned me, which among other
things, has the following device in it:

08:00.0 Ethernet controller: Intel Corporation I210 Gigabit Network Connection (rev 03)

This hotplugs just fine (kernel 4.2.0 plus a patch or two here):

[  863.020315] igb: Intel(R) Gigabit Ethernet Network Driver - version 5.2.18-k
[  863.020316] igb: Copyright (c) 2007-2014 Intel Corporation.
[  863.028657] igb 0000:08:00.0: enabling device (0000 -> 0002)
[  863.062089] igb 0000:08:00.0: added PHC on eth0
[  863.062090] igb 0000:08:00.0: Intel(R) Gigabit Ethernet Network Connection
[  863.062091] igb 0000:08:00.0: eth0: (PCIe:2.5Gb/s:Width x1) e8:ea:6a:00:1b:2a
[  863.062194] igb 0000:08:00.0: eth0: PBA No: 000200-000
[  863.062196] igb 0000:08:00.0: Using MSI-X interrupts. 4 rx queue(s), 4 tx queue(s)
[  863.064889] igb 0000:08:00.0 enp8s0: renamed from eth0

But disconnecting it is another story:

[ 1002.807932] igb 0000:08:00.0: removed PHC on enp8s0
[ 1002.807944] igb 0000:08:00.0 enp8s0: PCIe link lost, device now detached
[ 1003.341141] ------------[ cut here ]------------
[ 1003.341148] WARNING: CPU: 0 PID: 199 at lib/iomap.c:43 bad_io_access+0x38/0x40()
[ 1003.341149] Bad IO access at port 0x0 ()
[ 1003.342767] Modules linked in: snd_usb_audio snd_usbmidi_lib snd_rawmidi igb dca firewire_ohci firewire_core crc_itu_t rfcomm ctr ccm arc4 iwlmvm mac80211 fuse xt_CHECKSUM ipt_MASQUERADE
nf_nat_masquerade_ipv4 tun ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat
nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat
nf_conntrack iptable_mangle iptable_security iptable_raw iptable_filter bnep dm_mirror dm_region_hash dm_log dm_mod coretemp x86_pkg_temp_thermal intel_powerclamp kvm_intel snd_hda_codec_hdmi kvm
crct10dif_pclmul crc32_pclmul ghash_clmulni_intel drbg
[ 1003.342793]  ansi_cprng aesni_intel hp_wmi aes_x86_64 iTCO_wdt lrw iTCO_vendor_support ppdev gf128mul sparse_keymap glue_helper ablk_helper cryptd snd_hda_codec_realtek snd_hda_codec_generic
microcode snd_hda_intel uvcvideo iwlwifi snd_hda_codec videobuf2_vmalloc videobuf2_memops snd_hda_core videobuf2_core snd_hwdep btusb v4l2_common btrtl snd_seq btbcm btintel videodev cfg80211
snd_seq_device rtsx_pci_ms bluetooth pcspkr input_leds i2c_i801 media parport_pc memstick rfkill sg lpc_ich snd_pcm 8250_fintek parport joydev snd_timer snd soundcore hp_accel ie31200_edac
mei_me lis3lv02d edac_core input_polldev mei hp_wireless shpchp tpm_infineon sch_fq_codel nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables autofs4 xfs libcrc32c sd_mod sr_mod cdrom
rtsx_pci_sdmmc mmc_core crc32c_intel serio_raw rtsx_pci
[ 1003.342822]  nouveau ahci libahci mxm_wmi e1000e xhci_pci hwmon ptp drm_kms_helper pps_core xhci_hcd ttm wmi video ipv6
[ 1003.342839] CPU: 0 PID: 199 Comm: kworker/0:2 Not tainted 4.2.0-2.el7_UNSUPPORTED.x86_64 #1
[ 1003.342840] Hardware name: Hewlett-Packard HP ZBook 15 G2/2253, BIOS M70 Ver. 01.07 02/26/2015
[ 1003.342843] Workqueue: pciehp-3 pciehp_power_thread
[ 1003.342844]  ffffffff81a90655 ffff8804866d3b48 ffffffff8164763a 0000000000000000
[ 1003.342846]  ffff8804866d3b98 ffff8804866d3b88 ffffffff8107134a ffff8804866d3b88

49 more lines

diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index 212d668..df1d1c5 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -389,6 +389,8 @@ struct igb_adapter {
 	u16 link_speed;
 	u16 link_duplex;
 
+	u8 __iomem *io_addr; /* Mainly for iounmap use */
+
 	struct work_struct reset_task;
 	struct work_struct watchdog_task;
 	bool fc_autoneg;
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index e174fbb..76928b6 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -2294,9 +2294,11 @@ static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 
 	err = -EIO;
-	hw->hw_addr = pci_iomap(pdev, 0, 0);
-	if (!hw->hw_addr)
+	adapter->io_addr = pci_iomap(pdev, 0, 0);
+	if (!adapter->io_addr)
 		goto err_ioremap;
+	/* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
+	hw->hw_addr = adapter->io_addr;
 
 	netdev->netdev_ops = &igb_netdev_ops;
 	igb_set_ethtool_ops(netdev);
@@ -2656,7 +2658,7 @@ err_sw_init:
 #ifdef CONFIG_PCI_IOV
 	igb_disable_sriov(pdev);
 #endif
-	pci_iounmap(pdev, hw->hw_addr);
+	pci_iounmap(pdev, adapter->io_addr);
 err_ioremap:
 	free_netdev(netdev);
 err_alloc_etherdev:
@@ -2823,7 +2825,7 @@ static void igb_remove(struct pci_dev *pdev)
 
 	igb_clear_interrupt_scheme(adapter);
 
-	pci_iounmap(pdev, hw->hw_addr);
+	pci_iounmap(pdev, adapter->io_addr);
 	if (hw->flash_address)
 		iounmap(hw->flash_address);
 	pci_release_selected_regions(pdev,
-- 
1.8.3.1
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help