From: Peter Zijlstra <hidden> Date: 2008-07-24 14:42:37
With the introduction of the shared dirty page accounting in .19, NFS should
not be able to surpise the VM with all dirty pages. Thus it should always be
able to free some memory. Hence no more need for mempools.
Signed-off-by: Peter Zijlstra <redacted>
---
fs/nfs/read.c | 15 +++------------
fs/nfs/write.c | 27 +++++----------------------
2 files changed, 8 insertions(+), 34 deletions(-)
Index: linux-2.6/fs/nfs/read.c
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:43:13
Restrict objects from reserve slabs (ALLOC_NO_WATERMARKS) to allocation
contexts that are entitled to it. This is done to ensure reserve pages don't
leak out and get consumed.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/slub_def.h | 1
mm/slab.c | 60 +++++++++++++++++++++++++++++++++++++++--------
mm/slub.c | 28 ++++++++++++++++++---
3 files changed, 75 insertions(+), 14 deletions(-)
Index: linux-2.6/mm/slub.c
===================================================================
@@ -1509,10 +1513,20 @@ static void *__slab_alloc(struct kmem_ca{void**object;structpage*new;+intreserve;/* We handle __GFP_ZERO in the caller */gfpflags&=~__GFP_ZERO;+if(unlikely(c->reserve)){+/*+*Ifthecurrentslabisareserveslabandthecurrent+*allocationcontextdoesnotallowaccesstothereserveswe+*mustforceanallocationtotestthecurrentlevels.+*/+if(!(gfp_to_alloc_flags(gfpflags)&ALLOC_NO_WATERMARKS))+gotogrow_slab;+}if(!c->page)gotonew_slab;
@@ -38,6 +38,7 @@ struct kmem_cache_cpu {intnode;/* The node of the page (or -1 for debug) */unsignedintoffset;/* Freepointer offset (in word units) */unsignedintobjsize;/* Size of an object (from kmem_cache) */+intreserve;/* Did the current page come from the reserve */#ifdef CONFIG_SLUB_STATSunsignedstat[NR_SLUB_STAT_ITEMS];#endif
@@ -3044,11 +3077,14 @@ alloc_done:if(unlikely(!ac->avail)){intx;+force_grow:x=cache_grow(cachep,flags|GFP_THISNODE,node,NULL);/* cache_grow can reenable interrupts, then ac could change. */ac=cpu_cache_get(cachep);-if(!x&&ac->avail==0)/* no objects in sight? abort */++/* no objects in sight? abort */+if(!x&&(ac->avail==0||must_refill))returnNULL;if(!ac->avail)/* objects refilled by interrupt? */
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:43:43
Provide the basic infrastructure to reserve and charge/account network memory.
We provide the following reserve tree:
1) total network reserve
2) network TX reserve
3) protocol TX pages
4) network RX reserve
5) SKB data reserve
[1] is used to make all the network reserves a single subtree, for easy
manipulation.
[2] and [4] are merely for eastetic reasons.
The TX pages reserve [3] is assumed bounded by it being the upper bound of
memory that can be used for sending pages (not quite true, but good enough)
The SKB reserve [5] is an aggregate reserve, which is used to charge SKB data
against in the fallback path.
The consumers for these reserves are sockets marked with:
SOCK_MEMALLOC
Such sockets are to be used to service the VM (iow. to swap over). They
must be handled kernel side, exposing such a socket to user-space is a BUG.
Signed-off-by: Peter Zijlstra <redacted>
---
include/net/sock.h | 43 ++++++++++++++++++++-
net/Kconfig | 3 +
net/core/sock.c | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 152 insertions(+), 1 deletion(-)
Index: linux-2.6/include/net/sock.h
===================================================================
@@ -406,6 +407,7 @@ enum sock_flags {SOCK_RCVTSTAMPNS,/* %SO_TIMESTAMPNS setting */SOCK_LOCALROUTE,/* route locally only, %SO_DONTROUTE setting */SOCK_QUEUE_SHRUNK,/* write queue has been shrunk recently */+SOCK_MEMALLOC,/* the VM depends on us - make sure we're serviced */};staticinlinevoidsock_copy_flags(structsock*nsk,structsock*osk)
@@ -213,6 +214,105 @@ __u32 sysctl_rmem_default __read_mostly /* Maximal space eaten by iovec or ancilliary data plus some space */intsysctl_optmem_max__read_mostly=sizeof(unsignedlong)*(2*UIO_MAXIOV+512);+staticstructmem_reservenet_reserve;+structmem_reservenet_rx_reserve;+EXPORT_SYMBOL_GPL(net_rx_reserve);/* modular ipv6 only */+structmem_reservenet_skb_reserve;+EXPORT_SYMBOL_GPL(net_skb_reserve);/* modular ipv6 only */+staticstructmem_reservenet_tx_reserve;+staticstructmem_reservenet_tx_pages;++#ifdef CONFIG_NETVM+staticDEFINE_MUTEX(memalloc_socks_lock);+intmemalloc_socks;++/**+*sk_adjust_memalloc-adjusttheglobalmemallocreserveforcriticalRX+*@socks:numberofnew%SOCK_MEMALLOCsockets+*@tx_resserve_pages:numberofpagesto(un)reserveforTX+*+*Thisfunctionadjuststhememallocreservebasedonsystemdemand.+*TheRXreserveisalimit,andonlyaddedonce,notforeachsocket.+*+*NOTE:+*@tx_reserve_pagesisanupper-boundofmemoryusedforTXhence+*weneednotaccountthepageslikewedoforRXpages.+*/+intsk_adjust_memalloc(intsocks,longtx_reserve_pages)+{+interr;++mutex_lock(&memalloc_socks_lock);+err=mem_reserve_pages_add(&net_tx_pages,tx_reserve_pages);+if(err)+gotounlock;++/*+*eithersocksispositiveandweneedtocheckfor0->!0+*transitionandconnectthereservetreewhenweobserveit.+*/+if(!memalloc_socks&&socks>0){+err=mem_reserve_connect(&net_reserve,&mem_reserve_root);+if(err){+/*+*ifwefailedtoconnectthetree,undothetx+*reservesothatfailurehasnosideeffects.+*/+mem_reserve_pages_add(&net_tx_pages,-tx_reserve_pages);+gotounlock;+}+}+memalloc_socks+=socks;+/*+*orsocksisnegativeandwemustobservethe!0->0transition+*anddisconnectthereservetree.+*/+if(!memalloc_socks&&socks)+mem_reserve_disconnect(&net_reserve);++unlock:+mutex_unlock(&memalloc_socks_lock);++returnerr;+}+EXPORT_SYMBOL_GPL(sk_adjust_memalloc);++/**+*sk_set_memalloc-sets%SOCK_MEMALLOC+*@sk:sockettosetiton+*+*Set%SOCK_MEMALLOConasocketandincreasethememallocreserve+*accordingly.+*/+intsk_set_memalloc(structsock*sk)+{+intset=sock_flag(sk,SOCK_MEMALLOC);++if(!set){+interr=sk_adjust_memalloc(1,0);+if(err)+returnerr;++sock_set_flag(sk,SOCK_MEMALLOC);+sk->sk_allocation|=__GFP_MEMALLOC;+}+return!set;+}+EXPORT_SYMBOL_GPL(sk_set_memalloc);++intsk_clear_memalloc(structsock*sk)+{+intset=sock_flag(sk,SOCK_MEMALLOC);+if(set){+sk_adjust_memalloc(-1,0);+sock_reset_flag(sk,SOCK_MEMALLOC);+sk->sk_allocation&=~__GFP_MEMALLOC;+}+returnset;+}+EXPORT_SYMBOL_GPL(sk_clear_memalloc);+#endif+staticintsock_set_timeout(long*timeo_p,char__user*optval,intoptlen){structtimevaltv;
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:44:16
From: Neil Brown <redacted>
Document describing the problem and proposed solution
Signed-off-by: Peter Zijlstra <redacted>
---
Documentation/network-swap.txt | 270 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 270 insertions(+)
Index: linux-2.6/Documentation/network-swap.txt
===================================================================
@@ -0,0 +1,270 @@++Problem:+ When Linux needs to allocate memory it may find that there is+ insufficient free memory so it needs to reclaim space that is in+ use but not needed at the moment. There are several options:++ 1/ Shrink a kernel cache such as the inode or dentry cache. This+ is fairly easy but provides limited returns.+ 2/ Discard 'clean' pages from the page cache. This is easy, and+ works well as long as there are clean pages in the page cache.+ Similarly clean 'anonymous' pages can be discarded - if there+ are any.+ 3/ Write out some dirty page-cache pages so that they become clean.+ The VM limits the number of dirty page-cache pages to e.g. 40%+ of available memory so that (among other reasons) a "sync" will+ not take excessively long. So there should never be excessive+ amounts of dirty pagecache.+ Writing out dirty page-cache pages involves work by the+ filesystem which may need to allocate memory itself. To avoid+ deadlock, filesystems use GFP_NOFS when allocating memory on the+ write-out path. When this is used, cleaning dirty page-cache+ pages is not an option so if the filesystem finds that memory+ is tight, another option must be found.+ 4/ Write out dirty anonymous pages to the "Swap" partition/file.+ This is the most interesting for a couple of reasons.+ a/ Unlike dirty page-cache pages, there is no need to write anon+ pages out unless we are actually short of memory. Thus they+ tend to be left to last.+ b/ Anon pages tend to be updated randomly and unpredictably, and+ flushing them out of memory can have a very significant+ performance impact on the process using them. This contrasts+ with page-cache pages which are often written sequentially+ and often treated as "write-once, read-many".+ So anon pages tend to be left until last to be cleaned, and may+ be the only cleanable pages while there are still some dirty+ page-cache pages (which are waiting on a GFP_NOFS allocation).++[I don't find the above wholly satisfying. There seems to be too much+ hand-waving. If someone can provide better text explaining why+ swapout is a special case, that would be great.]++So we need to be able to write to the swap file/partition without+needing to allocate any memory ... or only a small well controlled+amount.++The VM reserves a small amount of memory that can only be allocated+for use as part of the swap-out procedure. It is only available to+processes with the PF_MEMALLOC flag set, which is typically just the+memory cleaner.++Traditionally swap-out is performed directly to block devices (swap+files on block-device filesystems are supported by examining the+mapping from file offset to device offset in advance, and then using+the device offsets to write directly to the device). Block devices+are (required to be) written to pre-allocate any memory that might be+needed during write-out, and to block when the pre-allocated memory is+exhausted and no other memory is available. They can be sure not to+block forever as the pre-allocated memory will be returned as soon as+the data it is being used for has been written out. The primary+mechanism for pre-allocating memory is called "mempools".++This approach does not work for writing anonymous pages+(i.e. swapping) over a network, using e.g NFS or NBD or iSCSI.+++The main reason that it does not work is that when data from an anon+page is written to the network, we must wait for a reply to confirm+the data is safe. Receiving that reply will consume memory and,+significantly, we need to allocate memory to an incoming packet before+we can tell if it is the reply we are waiting for or not.++The secondary reason is that the network code is not written to use+mempools and in most cases does not need to use them. Changing all+allocations in the networking layer to use mempools would be quite+intrusive, and would waste memory, and probably cause a slow-down in+the common case of not swapping over the network.++These problems are addressed by enhancing the system of memory+reserves used by PF_MEMALLOC and requiring any in-kernel networking+client that is used for swap-out to indicate which sockets are used+for swapout so they can be handled specially in low memory situations.++There are several major parts to this enhancement:++1/ page->reserve, GFP_MEMALLOC++ To handle low memory conditions we need to know when those+ conditions exist. Having a global "low on memory" flag seems easy,+ but its implementation is problematic. Instead we make it possible+ to tell if a recent memory allocation required use of the emergency+ memory pool.+ For pages returned by alloc_page, the new page->reserve flag+ can be tested. If this is set, then a low memory condition was+ current when the page was allocated, so the memory should be used+ carefully. (Because low memory conditions are transient, this+ state is kept in an overloaded member instead of in page flags, which+ would suggest a more permanent state.)++ For memory allocated using slab/slub: If a page that is added to a+ kmem_cache is found to have page->reserve set, then a s->reserve+ flag is set for the whole kmem_cache. Further allocations will only+ be returned from that page (or any other page in the cache) if they+ are emergency allocation (i.e. PF_MEMALLOC or GFP_MEMALLOC is set).+ Non-emergency allocations will block in alloc_page until a+ non-reserve page is available. Once a non-reserve page has been+ added to the cache, the s->reserve flag on the cache is removed.++ Because slab objects have no individual state its hard to pass+ reserve state along, the current code relies on a regular alloc+ failing. There are various allocation wrappers help here.++ This allows us to+ a/ request use of the emergency pool when allocating memory+ (GFP_MEMALLOC), and+ b/ to find out if the emergency pool was used.++2/ SK_MEMALLOC, sk_buff->emergency.++ When memory from the reserve is used to store incoming network+ packets, the memory must be freed (and the packet dropped) as soon+ as we find out that the packet is not for a socket that is used for+ swap-out.+ To achieve this we have an ->emergency flag for skbs, and an+ SK_MEMALLOC flag for sockets.+ When memory is allocated for an skb, it is allocated with+ GFP_MEMALLOC (if we are currently swapping over the network at+ all). If a subsequent test shows that the emergency pool was used,+ ->emergency is set.+ When the skb is finally attached to its destination socket, the+ SK_MEMALLOC flag on the socket is tested. If the skb has+ ->emergency set, but the socket does not have SK_MEMALLOC set, then+ the skb is immediately freed and the packet is dropped.+ This ensures that reserve memory is never queued on a socket that is+ not used for swapout.++ Similarly, if an skb is ever queued for delivery to user-space for+ example by netfilter, the ->emergency flag is tested and the skb is+ released if ->emergency is set. (so obviously the storage route may+ not pass through a userspace helper, otherwise the packets will never+ arrive and we'll deadlock)++ This ensures that memory from the emergency reserve can be used to+ allow swapout to proceed, but will not get caught up in any other+ network queue.+++3/ pages_emergency++ The above would be sufficient if the total memory below the lowest+ memory watermark (i.e the size of the emergency reserve) were known+ to be enough to hold all transient allocations needed for writeout.+ I'm a little blurry on how big the current emergency pool is, but it+ isn't big and certainly hasn't been sized to allow network traffic+ to consume any.++ We could simply make the size of the reserve bigger. However in the+ common case that we are not swapping over the network, that would be+ a waste of memory.++ So a new "watermark" is defined: pages_emergency. This is+ effectively added to the current low water marks, so that pages from+ this emergency pool can only be allocated if one of PF_MEMALLOC or+ GFP_MEMALLOC are set.++ pages_emergency can be changed dynamically based on need. When+ swapout over the network is required, pages_emergency is increased+ to cover the maximum expected load. When network swapout is+ disabled, pages_emergency is decreased.++ To determine how much to increase it by, we introduce reservation+ groups....++3a/ reservation groups++ The memory used transiently for swapout can be in a number of+ different places. e.g. the network route cache, the network+ fragment cache, in transit between network card and socket, or (in+ the case of NFS) in sunrpc data structures awaiting a reply.+ We need to ensure each of these is limited in the amount of memory+ they use, and that the maximum is included in the reserve.++ The memory required by the network layer only needs to be reserved+ once, even if there are multiple swapout paths using the network+ (e.g. NFS and NDB and iSCSI, though using all three for swapout at+ the same time would be unusual).++ So we create a tree of reservation groups. The network might+ register a collection of reservations, but not mark them as being in+ use. NFS and sunrpc might similarly register a collection of+ reservations, and attach it to the network reservations as it+ depends on them.+ When swapout over NFS is requested, the NFS/sunrpc reservations are+ activated which implicitly activates the network reservations.++ The total new reservation is added to pages_emergency.++ Provided each memory usage stays beneath the registered limit (at+ least when allocating memory from reserves), the system will never+ run out of emergency memory, and swapout will not deadlock.++ It is worth noting here that it is not critical that each usage+ stays beneath the limit 100% of the time. Occasional excess is+ acceptable provided that the memory will be freed again within a+ short amount of time that does *not* require waiting for any event+ that itself might require memory.+ This is because, at all stages of transmit and receive, it is+ acceptable to discard all transient memory associated with a+ particular writeout and try again later. On transmit, the page can+ be re-queued for later transmission. On receive, the packet can be+ dropped assuming that the peer will resend after a timeout.++ Thus allocations that are truly transient and will be freed without+ blocking do not strictly need to be reserved for. Doing so might+ still be a good idea to ensure forward progress doesn't take too+ long.++4/ low-mem accounting++ Most places that might hold on to emergency memory (e.g. route+ cache, fragment cache etc) already place a limit on the amount of+ memory that they can use. This limit can simply be reserved using+ the above mechanism and no more needs to be done.++ However some memory usage might not be accounted with sufficient+ firmness to allow an appropriate emergency reservation. The+ in-flight skbs for incoming packets is one such example.++ To support this, a low-overhead mechanism for accounting memory+ usage against the reserves is provided. This mechanism uses the+ same data structure that is used to store the emergency memory+ reservations through the addition of a 'usage' field.++ Before we attempt allocation from the memory reserves, we much check+ if the resulting 'usage' is below the reservation. If so, we increase+ the usage and attempt the allocation (which should succeed). If+ the projected 'usage' exceeds the reservation we'll either fail the+ allocation, or wait for 'usage' to decrease enough so that it would+ succeed, depending on __GFP_WAIT.++ When memory that was allocated for that purpose is freed, the+ 'usage' field is checked again. If it is non-zero, then the size of+ the freed memory is subtracted from the usage, making sure the usage+ never becomes less than zero.++ This provides adequate accounting with minimal overheads when not in+ a low memory condition. When a low memory condition is encountered+ it does add the cost of a spin lock necessary to serialise updates+ to 'usage'.++++5/ swapon/swapoff/swap_out/swap_in++ So that a filesystem (e.g. NFS) can know when to set SK_MEMALLOC on+ any network socket that it uses, and can know when to account+ reserve memory carefully, new address_space_operations are+ available.+ "swapon" requests that an address space (i.e a file) be make ready+ for swapout. swap_out and swap_in request the actual IO. They+ together must ensure that each swap_out request can succeed without+ allocating more emergency memory that was reserved by swapon. swapoff+ is used to reverse the state changes caused by swapon when we disable+ the swap file.+++Thanks for reading this far. I hope it made sense :-)++Neil Brown (with updates from Peter Zijlstra)++
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:44:48
Provide a method to get the upper bound on the pages needed to allocate
a given number of objects from a given kmem_cache.
This lays the foundation for a generic reserve framework as presented in
a later patch in this series. This framework needs to convert object demand
(kmalloc() bytes, kmem_cache_alloc() objects) to pages.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/slab.h | 4 ++
mm/slab.c | 75 +++++++++++++++++++++++++++++++++++++++++++
mm/slub.c | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 166 insertions(+)
Index: linux-2.6/include/linux/slab.h
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:45:16
Avoid memory getting stuck waiting for userspace, drop all emergency packets.
This of course requires the regular storage route to not include an NF_QUEUE
target ;-)
Signed-off-by: Peter Zijlstra <redacted>
---
net/netfilter/core.c | 3 +++
1 file changed, 3 insertions(+)
Index: linux-2.6/net/netfilter/core.c
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:45:44
In order to teach filesystems to handle swap cache pages, three new page
functions are introduced:
pgoff_t page_file_index(struct page *);
loff_t page_file_offset(struct page *);
struct address_space *page_file_mapping(struct page *);
page_file_index() - gives the offset of this page in the file in
PAGE_CACHE_SIZE blocks. Like page->index is for mapped pages, this function
also gives the correct index for PG_swapcache pages.
page_file_offset() - uses page_file_index(), so that it will give the expected
result, even for PG_swapcache pages.
page_file_mapping() - gives the mapping backing the actual page; that is for
swap cache pages it will give swap_file->f_mapping.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/mm.h | 25 +++++++++++++++++++++++++
include/linux/pagemap.h | 5 +++++
mm/swapfile.c | 19 +++++++++++++++++++
3 files changed, 49 insertions(+)
Index: linux-2.6/include/linux/mm.h
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:46:17
The reserve is proportionally distributed over all (!highmem) zones in the
system. So we need to allow an emergency allocation access to all zones. In
order to do that we need to break out of any mempolicy boundaries we might
have.
In my opinion that does not break mempolicies as those are user oriented
and not system oriented. That is, system allocations are not guaranteed to be
within mempolicy boundaries. For instance IRQs don't even have a mempolicy.
So breaking out of mempolicy boundaries for 'rare' emergency allocations,
which are always system allocations (as opposed to user) is ok.
Signed-off-by: Peter Zijlstra <redacted>
---
mm/page_alloc.c | 5 +++++
1 file changed, 5 insertions(+)
Index: linux-2.6/mm/page_alloc.c
===================================================================
@@ -1533,6 +1533,11 @@ restart:rebalance:if(alloc_flags&ALLOC_NO_WATERMARKS){nofail_alloc:+/*+*breakoutofmempolicyboundaries+*/+zonelist=node_zonelist(numa_node_id(),gfp_mask);+/* go through the zonelist yet again, ignoring mins */page=get_page_from_freelist(gfp_mask,nodemask,order,zonelist,high_zoneidx,ALLOC_NO_WATERMARKS);
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:46:39
It could happen that all !SOCK_MEMALLOC sockets have buffered so much data
that we're over the global rmem limit. This will prevent SOCK_MEMALLOC buffers
from receiving data, which will prevent userspace from running, which is needed
to reduce the buffered data.
Fix this by exempting the SOCK_MEMALLOC sockets from the rmem limit.
Signed-off-by: Peter Zijlstra <redacted>
---
include/net/sock.h | 7 ++++---
net/core/sock.c | 2 +-
net/ipv4/tcp_input.c | 12 ++++++------
net/sctp/ulpevent.c | 2 +-
4 files changed, 12 insertions(+), 11 deletions(-)
Index: linux-2.6/include/net/sock.h
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:47:18
Tag pages allocated from the reserves with a non-zero page->reserve.
This allows us to distinguish and account reserve pages.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/mm_types.h | 1 +
mm/page_alloc.c | 4 +++-
2 files changed, 4 insertions(+), 1 deletion(-)
Index: linux-2.6/include/linux/mm_types.h
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:47:37
There is a small race between the procfs caller and the memory hotplug caller
of setup_per_zone_pages_min(). Not a big deal, but the next patch will add yet
another caller. Time to close the gap.
Signed-off-by: Peter Zijlstra <redacted>
---
mm/page_alloc.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
Index: linux-2.6/mm/page_alloc.c
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:47:54
Provide means to reserve a specific amount of pages.
The emergency pool is separated from the min watermark because ALLOC_HARDER
and ALLOC_HIGH modify the watermark in a relative way and thus do not ensure
a strict minimum.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/mmzone.h | 6 ++-
mm/page_alloc.c | 84 +++++++++++++++++++++++++++++++++++++++++++------
mm/vmstat.c | 6 +--
3 files changed, 82 insertions(+), 14 deletions(-)
Index: linux-2.6/include/linux/mmzone.h
===================================================================
@@ -265,7 +265,10 @@ enum zone_type {structzone{/* Fields commonly accessed by the page allocator */-unsignedlongpages_min,pages_low,pages_high;+unsignedlongpages_high;/* we stop kswapd */+unsignedlongpages_low;/* we wake up kswapd */+unsignedlongpages_min;/* we enter direct reclaim */+unsignedlongpages_emerg;/* emergency pool *//**Wedon'tknowifthememorythatwe'regoingtoallocatewillbefreeable*or/anditwillbereleasedeventually,sotoavoidtotallywastingseveral
@@ -751,6 +754,7 @@ int sysctl_min_unmapped_ratio_sysctl_hanstructfile*,void__user*,size_t*,loff_t*);intsysctl_min_slab_ratio_sysctl_handler(structctl_table*,int,structfile*,void__user*,size_t*,loff_t*);+intadjust_memalloc_reserve(intpages);externintnuma_zonelist_order_handler(structctl_table*,int,structfile*,void__user*,size_t*,loff_t*);
@@ -1235,7 +1237,7 @@ int zone_watermark_ok(struct zone *z, inif(alloc_flags&ALLOC_HARDER)min-=min/4;-if(free_pages<=min+z->lowmem_reserve[classzone_idx])+if(free_pages<=min+z->lowmem_reserve[classzone_idx]+z->pages_emerg)return0;for(o=0;o<order;o++){/* At the next order, this order's pages become unavailable */
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:48:18
Factor out the gfp to alloc_flags mapping so it can be used in other places.
Signed-off-by: Peter Zijlstra <redacted>
---
mm/internal.h | 10 +++++
mm/page_alloc.c | 95 +++++++++++++++++++++++++++++++-------------------------
2 files changed, 64 insertions(+), 41 deletions(-)
Index: linux-2.6/mm/internal.h
===================================================================
@@ -1567,49 +1597,28 @@ restart:*OK,we'rebelowthekswapdwatermarkandhavekickedbackground*reclaim.Nowthingsgetmorecomplex,sosetupalloc_flagsaccording*tohowwewanttoproceed.-*-*Thecallermaydipintopagereservesabitmoreifthecaller-*cannotrundirectreclaim,orifthecallerhasrealtimescheduling-*policyorisaskingfor__GFP_HIGHmemory.GFP_ATOMICrequestswill-*setbothALLOC_HARDER(!wait)andALLOC_HIGH(__GFP_HIGH).*/-alloc_flags=ALLOC_WMARK_MIN;-if((unlikely(rt_task(p))&&!in_interrupt())||!wait)-alloc_flags|=ALLOC_HARDER;-if(gfp_mask&__GFP_HIGH)-alloc_flags|=ALLOC_HIGH;-if(wait)-alloc_flags|=ALLOC_CPUSET;+alloc_flags=gfp_to_alloc_flags(gfp_mask);-/*-*Gothroughthezonelistagain.Let__GFP_HIGHandallocations-*comingfromrealtimetasksgodeeperintoreserves.-*-*Thisisthelastchance,ingeneral,beforethegotonopage.-*IgnorecpusetifGFP_ATOMIC(!wait)ratherthanfailalloc.-*Seealsocpuset_zone_allowed()commentinkernel/cpuset.c.-*/+/* This is the last chance, in general, before the goto nopage. */page=get_page_from_freelist(gfp_mask,nodemask,order,zonelist,-high_zoneidx,alloc_flags);+high_zoneidx,alloc_flags&~ALLOC_NO_WATERMARKS);if(page)gotogot_pg;/* This allocation should allow future memory freeing. */-rebalance:-if(((p->flags&PF_MEMALLOC)||unlikely(test_thread_flag(TIF_MEMDIE)))-&&!in_interrupt()){-if(!(gfp_mask&__GFP_NOMEMALLOC)){+if(alloc_flags&ALLOC_NO_WATERMARKS){nofail_alloc:-/* go through the zonelist yet again, ignoring mins */-page=get_page_from_freelist(gfp_mask,nodemask,order,+/* go through the zonelist yet again, ignoring mins */+page=get_page_from_freelist(gfp_mask,nodemask,order,zonelist,high_zoneidx,ALLOC_NO_WATERMARKS);-if(page)-gotogot_pg;-if(gfp_mask&__GFP_NOFAIL){-congestion_wait(WRITE,HZ/50);-gotonofail_alloc;-}+if(page)+gotogot_pg;++if(wait&&(gfp_mask&__GFP_NOFAIL)){+congestion_wait(WRITE,HZ/50);+gotonofail_alloc;}gotonopage;}
@@ -1618,6 +1627,10 @@ nofail_alloc:if(!wait)gotonopage;+/* Avoid recursion of direct reclaim */+if(p->flags&PF_MEMALLOC)+gotonopage;+cond_resched();/* We now go into synchronous reclaim */
@@ -2088,7 +2088,8 @@ void tcp_send_fin(struct sock *sk)}else{/* Socket is locked, keep trying until memory is available. */for(;;){-skb=alloc_skb_fclone(MAX_TCP_HEADER,GFP_KERNEL);+skb=alloc_skb_fclone(MAX_TCP_HEADER,+sk_allocation(sk,GFP_KERNEL));if(skb)break;yield();
@@ -2114,7 +2115,7 @@ void tcp_send_active_reset(struct sock *structsk_buff*skb;/* NOTE: No TCP options attached and we never retransmit this. */-skb=alloc_skb(MAX_TCP_HEADER,priority);+skb=alloc_skb(MAX_TCP_HEADER,sk_allocation(sk,priority));if(!skb){NET_INC_STATS(LINUX_MIB_TCPABORTFAILED);return;
@@ -580,7 +580,8 @@ static int tcp_v6_md5_do_add(struct sock}else{/* reallocate new list if current one is full. */if(!tp->md5sig_info){-tp->md5sig_info=kzalloc(sizeof(*tp->md5sig_info),GFP_ATOMIC);+tp->md5sig_info=kzalloc(sizeof(*tp->md5sig_info),+sk_allocation(sk,GFP_ATOMIC));if(!tp->md5sig_info){kfree(newkey);return-ENOMEM;
@@ -593,7 +594,8 @@ static int tcp_v6_md5_do_add(struct sock}if(tp->md5sig_info->alloced6==tp->md5sig_info->entries6){keys=kmalloc((sizeof(tp->md5sig_info->keys6[0])*-(tp->md5sig_info->entries6+1)),GFP_ATOMIC);+(tp->md5sig_info->entries6+1)),+sk_allocation(sk,GFP_ATOMIC));if(!keys){tcp_free_md5sig_pool();
@@ -717,7 +719,8 @@ static int tcp_v6_parse_md5_keys (structstructtcp_sock*tp=tcp_sk(sk);structtcp_md5sig_info*p;-p=kzalloc(sizeof(structtcp_md5sig_info),GFP_KERNEL);+p=kzalloc(sizeof(structtcp_md5sig_info),+sk_allocation(sk,GFP_KERNEL));if(!p)return-ENOMEM;
@@ -637,7 +637,8 @@ struct sk_buff *sk_stream_alloc_skb(stru/* The TCP header must be at least 32-bit aligned. */size=ALIGN(size,4);-skb=alloc_skb_fclone(size+sk->sk_prot->max_header,gfp);+skb=alloc_skb_fclone(size+sk->sk_prot->max_header,+sk_allocation(sk,gfp));if(skb){if(sk_wmem_schedule(sk,skb->truesize)){/*
@@ -163,11 +163,11 @@ static struct notifier_block slab_notifi#endifstaticenum{-DOWN,/* No slab functionality available */+DOWN=0,/* No slab functionality available */PARTIAL,/* kmem_cache_open() works but kmalloc does not */UP,/* Everything works but does not show up in sysfs */SYSFS/* Sysfs up */-}slab_state=DOWN;+}slab_state;/* A list of all slab caches on the system */staticDECLARE_RWSEM(slub_lock);
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:49:11
Do as Trond suggested:
http://lkml.org/lkml/2006/8/25/348
Disable NFS data cache revalidation on swap files since it doesn't really
make sense to have other clients change the file while you are using it.
Thereby we can stop setting PG_private on swap pages, since there ought to
be no further races with invalidate_inode_pages2() to deal with.
And since we cannot set PG_private we cannot use page->private (which is
already used by PG_swapcache pages anyway) to store the nfs_page. Thus
augment the new nfs_page_find_request logic.
Signed-off-by: Peter Zijlstra <redacted>
---
fs/nfs/inode.c | 6 ++++
fs/nfs/write.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++-----------
2 files changed, 64 insertions(+), 13 deletions(-)
Index: linux-2.6/fs/nfs/inode.c
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:49:35
Allow PF_MEMALLOC to be set in softirq context.
This is needed to allow network softirq packet processing to make use of
PF_MEMALLOC.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/sched.h | 7 +++++++
kernel/softirq.c | 3 +++
mm/page_alloc.c | 7 ++++---
3 files changed, 14 insertions(+), 3 deletions(-)
Index: linux-2.6/mm/page_alloc.c
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:49:56
Generic reserve management code.
It provides methods to reserve and charge. Upon this, generic alloc/free style
reserve pools could be build, which could fully replace mempool_t
functionality.
It should also allow for a Banker's algorithm replacement of __GFP_NOFAIL.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/reserve.h | 146 +++++++++++
include/linux/slab.h | 20 -
mm/Makefile | 2
mm/reserve.c | 594 ++++++++++++++++++++++++++++++++++++++++++++++++
mm/slub.c | 4
5 files changed, 755 insertions(+), 11 deletions(-)
Index: linux-2.6/include/linux/reserve.h
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:50:44
Change the skb allocation api to indicate RX usage and use this to fall back to
the reserve when needed. SKBs allocated from the reserve are tagged in
skb->emergency.
Teach all other skb ops about emergency skbs and the reserve accounting.
Use the (new) packet split API to allocate and track fragment pages from the
emergency reserve. Do this using an atomic counter in page->index. This is
needed because the fragments have a different sharing semantic than that
indicated by skb_shinfo()->dataref.
Note that the decision to distinguish between regular and emergency SKBs allows
the accounting overhead to be limited to the later kind.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/mm_types.h | 1
include/linux/skbuff.h | 25 +++++++--
net/core/skbuff.c | 129 +++++++++++++++++++++++++++++++++++++----------
3 files changed, 124 insertions(+), 31 deletions(-)
Index: linux-2.6/include/linux/skbuff.h
===================================================================
@@ -179,23 +179,29 @@ EXPORT_SYMBOL(skb_truesize_bug);*%GFP_ATOMIC.*/structsk_buff*__alloc_skb(unsignedintsize,gfp_tgfp_mask,-intfclone,intnode)+intflags,intnode){structkmem_cache*cache;structskb_shared_info*shinfo;structsk_buff*skb;u8*data;+intemergency=0;+intmemalloc=sk_memalloc_socks();-cache=fclone?skbuff_fclone_cache:skbuff_head_cache;+size=SKB_DATA_ALIGN(size);+cache=(flags&SKB_ALLOC_FCLONE)+?skbuff_fclone_cache:skbuff_head_cache;++if(memalloc&&(flags&SKB_ALLOC_RX))+gfp_mask|=__GFP_MEMALLOC;/* Get the HEAD */skb=kmem_cache_alloc_node(cache,gfp_mask&~__GFP_DMA,node);if(!skb)gotoout;-size=SKB_DATA_ALIGN(size);-data=kmalloc_node_track_caller(size+sizeof(structskb_shared_info),-gfp_mask,node);+data=kmalloc_reserve(size+sizeof(structskb_shared_info),+gfp_mask,node,&net_skb_reserve,&emergency);if(!data)gotonodata;
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:51:04
Toss all emergency packets not for a SOCK_MEMALLOC socket. This ensures our
precious memory reserve doesn't get stuck waiting for user-space.
The correctness of this approach relies on the fact that networks must be
assumed lossy.
Signed-off-by: Peter Zijlstra <redacted>
---
net/core/filter.c | 3 +++
1 file changed, 3 insertions(+)
Index: linux-2.6/net/core/filter.c
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:51:28
Implement all the new swapfile a_ops for NFS. This will set the NFS socket to
SOCK_MEMALLOC and run socket reconnect under PF_MEMALLOC as well as reset
SOCK_MEMALLOC before engaging the protocol ->connect() method.
PF_MEMALLOC should allow the allocation of struct socket and related objects
and the early (re)setting of SOCK_MEMALLOC should allow us to receive the
packets required for the TCP connection buildup.
(swapping continues over a server reset during heavy network traffic)
Signed-off-by: Peter Zijlstra <redacted>
---
fs/Kconfig | 17 ++++++++++
fs/nfs/file.c | 18 ++++++++++
fs/nfs/write.c | 22 +++++++++++++
include/linux/nfs_fs.h | 2 +
include/linux/sunrpc/xprt.h | 5 ++-
net/sunrpc/sched.c | 9 ++++-
net/sunrpc/xprtsock.c | 73 ++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 143 insertions(+), 3 deletions(-)
Index: linux-2.6/fs/nfs/file.c
===================================================================
@@ -147,7 +147,9 @@ struct rpc_xprt {unsignedintmax_reqs;/* total slots */unsignedlongstate;/* transport state */unsignedcharshutdown:1,/* being shut down */-resvport:1;/* use a reserved port */+resvport:1,/* use a reserved port */+swapper:1;/* we're swapping over this+transport*/unsignedintbind_index;/* bind function index *//*
@@ -1486,11 +1537,15 @@ static void xs_udp_connect_worker4(struccontainer_of(work,structsock_xprt,connect_worker.work);structrpc_xprt*xprt=&transport->xprt;structsocket*sock=transport->sock;+unsignedlongpflags=current->flags;interr,status=-EIO;if(xprt->shutdown||!xprt_bound(xprt))gotoout;+if(xprt->swapper)+current->flags|=PF_MEMALLOC;+/* Start by resetting any existing state */xs_close(xprt);
@@ -1527,11 +1583,15 @@ static void xs_udp_connect_worker6(struccontainer_of(work,structsock_xprt,connect_worker.work);structrpc_xprt*xprt=&transport->xprt;structsocket*sock=transport->sock;+unsignedlongpflags=current->flags;interr,status=-EIO;if(xprt->shutdown||!xprt_bound(xprt))gotoout;+if(xprt->swapper)+current->flags|=PF_MEMALLOC;+/* Start by resetting any existing state */xs_close(xprt);
@@ -1613,6 +1674,8 @@ static int xs_tcp_finish_connecting(struwrite_unlock_bh(&sk->sk_callback_lock);}+xs_set_memalloc(xprt);+/* Tell the socket layer to start connecting... */xprt->stat.connect_count++;xprt->stat.connect_start=jiffies;
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:51:45
Add reserves for INET.
The two big users seem to be the route cache and ip-fragment cache.
Reserve the route cache under generic RX reserve, its usage is bounded by
the high reclaim watermark, and thus does not need further accounting.
Reserve the ip-fragement caches under SKB data reserve, these add to the
SKB RX limit. By ensuring we can at least receive as much data as fits in
the reassmbly line we avoid fragment attack deadlocks.
Adds to the reserve tree:
total network reserve
network TX reserve
protocol TX pages
network RX reserve
+ IPv6 route cache
+ IPv4 route cache
SKB data reserve
+ IPv6 fragment cache
+ IPv4 fragment cache
Signed-off-by: Peter Zijlstra <redacted>
---
include/net/inet_frag.h | 7 +++
include/net/netns/ipv6.h | 4 ++
net/ipv4/inet_fragment.c | 3 +
net/ipv4/ip_fragment.c | 89 +++++++++++++++++++++++++++++++++++++++++++++--
net/ipv4/route.c | 72 +++++++++++++++++++++++++++++++++++++-
net/ipv6/af_inet6.c | 20 +++++++++-
net/ipv6/reassembly.c | 88 +++++++++++++++++++++++++++++++++++++++++++++-
net/ipv6/route.c | 66 ++++++++++++++++++++++++++++++++++
8 files changed, 341 insertions(+), 8 deletions(-)
Index: linux-2.6/net/ipv4/ip_fragment.c
===================================================================
@@ -42,6 +42,8 @@#include<linux/udp.h>#include<linux/inet.h>#include<linux/netfilter_ipv4.h>+#include<linux/reserve.h>+#include<linux/nsproxy.h>/* NOTE. Logic of IP defragmentation is parallel to corresponding IPv6*codenow.Ifyouchangesomethinghere,_PLEASE_updateipv6/reassembly.c
@@ -725,11 +790,31 @@ static int ipv4_frags_init_net(struct neinet_frags_init_net(&net->ipv4.frags);-returnip4_frags_ns_ctl_register(net);+ret=ip4_frags_ns_ctl_register(net);+if(ret)+gotoout_reg;++mem_reserve_init(&net->ipv4.frags.reserve,"IPv4 fragment cache",+&net_skb_reserve);+ret=mem_reserve_kmalloc_set(&net->ipv4.frags.reserve,+net->ipv4.frags.high_thresh);+if(ret)+gotoout_reserve;++return0;++out_reserve:+mem_reserve_disconnect(&net->ipv4.frags.reserve);+ip4_frags_ns_ctl_unregister(net);+out_reg:+inet_frags_exit_net(&net->ipv4.frags,&ip4_frags);++returnret;}staticvoidipv4_frags_exit_net(structnet*net){+mem_reserve_disconnect(&net->ipv4.frags.reserve);ip4_frags_ns_ctl_unregister(net);inet_frags_exit_net(&net->ipv4.frags,&ip4_frags);}
@@ -851,6 +851,20 @@ static int inet6_net_init(struct net *nenet->ipv6.sysctl.ip6_rt_min_advmss=IPV6_MIN_MTU-20-40;net->ipv6.sysctl.icmpv6_time=1*HZ;+mem_reserve_init(&net->ipv6.ip6_rt_reserve,"IPv6 route cache",+&net_rx_reserve);+/*+*XXX:requiresthatnet->ipv6.ip6_dst_opsisalreadyset-up+*butafaiktitsimpossibletoorderthevarious+*pernet_subsyscallssothatthisoneisdoneafter+*ip6_route_net_init().+*/+err=mem_reserve_kmem_cache_set(&net->ipv6.ip6_rt_reserve,+net->ipv6.ip6_dst_ops.kmem_cachep,+net->ipv6.sysctl.ip6_rt_max_size);+if(err)+gotoreserve_fail;+#ifdef CONFIG_PROC_FSerr=udp6_proc_init(net);if(err)
@@ -861,8 +875,8 @@ static int inet6_net_init(struct net *neerr=ac6_proc_init(net);if(err)gotoproc_ac6_fail;-out:#endif+out:returnerr;#ifdef CONFIG_PROC_FS
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:52:05
In order to make sure emergency packets receive all memory needed to proceed
ensure processing of emergency SKBs happens under PF_MEMALLOC.
Use the (new) sk_backlog_rcv() wrapper to ensure this for backlog processing.
Skip taps, since those are user-space again.
Signed-off-by: Peter Zijlstra <redacted>
---
include/net/sock.h | 5 ++++
net/core/dev.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++------
net/core/sock.c | 16 ++++++++++++++
3 files changed, 74 insertions(+), 6 deletions(-)
Index: linux-2.6/net/core/dev.c
===================================================================
@@ -2029,10 +2053,23 @@ int netif_receive_skb(struct sk_buff *skstructnet_device*orig_dev;intret=NET_RX_DROP;__be16type;+unsignedlongpflags=current->flags;++/* Emergency skb are special, they should+*-bedeliveredtoSOCK_MEMALLOCsocketsonly+*-stayawayfromuserspace+*-haveboundedmemoryusage+*+*UsePF_MEMALLOCasapoormansmemorypool-thegroupingkind.+*Thissavesusfrompropagatingtheallocationcontextdowntoall+*allocationsites.+*/+if(skb_emergency(skb))+current->flags|=PF_MEMALLOC;/* if we've gotten here through NAPI, check netpoll */if(netpoll_receive_skb(skb))-returnNET_RX_DROP;+gotoout;if(!skb->tstamp.tv64)net_timestamp(skb);
@@ -2043,7 +2080,7 @@ int netif_receive_skb(struct sk_buff *skorig_dev=skb_bond(skb);if(!orig_dev)-returnNET_RX_DROP;+gotoout;__get_cpu_var(netdev_rx_stat).total++;
@@ -2062,6 +2099,9 @@ int netif_receive_skb(struct sk_buff *sk}#endif+if(skb_emergency(skb))+gotoskip_taps;+list_for_each_entry_rcu(ptype,&ptype_all,list){if(!ptype->dev||ptype->dev==skb->dev){if(pt_prev)
@@ -2070,19 +2110,23 @@ int netif_receive_skb(struct sk_buff *sk}}+skip_taps:#ifdef CONFIG_NET_CLS_ACTskb=handle_ing(skb,&pt_prev,&ret,orig_dev);if(!skb)-gotoout;+gotounlock;ncls:#endif+if(!skb_emergency_protocol(skb))+gotodrop;+skb=handle_bridge(skb,&pt_prev,&ret,orig_dev);if(!skb)-gotoout;+gotounlock;skb=handle_macvlan(skb,&pt_prev,&ret,orig_dev);if(!skb)-gotoout;+gotounlock;type=skb->protocol;list_for_each_entry_rcu(ptype,
@@ -2098,6 +2142,7 @@ ncls:if(pt_prev){ret=pt_prev->func(skb,skb->dev,pt_prev,orig_dev);}else{+drop:kfree_skb(skb);/* Jamal, now you will not able to escape explaining*mehowyouweregoingtousethis.:-)
@@ -313,6 +313,22 @@ int sk_clear_memalloc(struct sock *sk)returnset;}EXPORT_SYMBOL_GPL(sk_clear_memalloc);++int__sk_backlog_rcv(structsock*sk,structsk_buff*skb)+{+intret;+unsignedlongpflags=current->flags;++/* these should have been dropped before queueing */+BUG_ON(!sk_has_memalloc(sk));++current->flags|=PF_MEMALLOC;+ret=sk->sk_backlog_rcv(sk,skb);+tsk_restore_flags(current,pflags,PF_MEMALLOC);++returnret;+}+EXPORT_SYMBOL(__sk_backlog_rcv);#endifstaticintsock_set_timeout(long*timeo_p,char__user*optval,intoptlen)
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:52:41
GFP_NOFS is not enough, since swap traffic is IO, hence fall back to GFP_NOIO.
Signed-off-by: Peter Zijlstra <redacted>
---
fs/nfs/pagelist.c | 2 +-
fs/nfs/write.c | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
Index: linux-2.6/fs/nfs/write.c
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:53:01
Add some packet-split receive hooks.
For one this allows to do NUMA node affine page allocs. Later on these hooks
will be extended to do emergency reserve allocations for fragments.
Signed-off-by: Peter Zijlstra <redacted>
---
drivers/net/bnx2.c | 8 +++-----
drivers/net/e1000/e1000_main.c | 8 ++------
drivers/net/e1000e/netdev.c | 7 ++-----
drivers/net/igb/igb_main.c | 8 ++------
drivers/net/ixgbe/ixgbe_main.c | 10 +++-------
drivers/net/sky2.c | 16 ++++++----------
include/linux/skbuff.h | 23 +++++++++++++++++++++++
net/core/skbuff.c | 20 ++++++++++++++++++++
8 files changed, 61 insertions(+), 39 deletions(-)
Index: linux-2.6/drivers/net/e1000/e1000_main.c
===================================================================
@@ -4492,12 +4492,8 @@ e1000_clean_rx_irq_ps(struct e1000_adaptpci_unmap_page(pdev,ps_page_dma->ps_page_dma[j],PAGE_SIZE,PCI_DMA_FROMDEVICE);ps_page_dma->ps_page_dma[j]=0;-skb_fill_page_desc(skb,j,ps_page->ps_page[j],0,-length);+skb_add_rx_frag(skb,j,ps_page->ps_page[j],0,length);ps_page->ps_page[j]=NULL;-skb->len+=length;-skb->data_len+=length;-skb->truesize+=length;}/* strip the ethernet crc, problem is we're using pages now so
@@ -2217,8 +2217,8 @@ static struct sk_buff *receive_copy(stru}/* Adjust length of skb with fragments to match received data */-staticvoidskb_put_frags(structsk_buff*skb,unsignedinthdr_space,-unsignedintlength)+staticvoidskb_put_frags(structsky2_port*sky2,structsk_buff*skb,+unsignedinthdr_space,unsignedintlength){inti,num_frags;unsignedintsize;
@@ -2235,15 +2235,11 @@ static void skb_put_frags(struct sk_buffif(length==0){/* don't need this page */-__free_page(frag->page);+netdev_free_page(sky2->netdev,frag->page);--skb_shinfo(skb)->nr_frags;}else{size=min(length,(unsigned)PAGE_SIZE);--frag->size=size;-skb->data_len+=size;-skb->truesize+=size;-skb->len+=size;+skb_add_rx_frag(skb,i,frag->page,0,size);length-=size;}}
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:53:23
__GFP_MEMALLOC will allow the allocation to disregard the watermarks,
much like PF_MEMALLOC.
It allows one to pass along the memalloc state in object related allocation
flags as opposed to task related flags, such as sk->sk_allocation.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/gfp.h | 3 ++-
mm/page_alloc.c | 4 +++-
2 files changed, 5 insertions(+), 2 deletions(-)
Index: linux-2.6/include/linux/gfp.h
===================================================================
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:53:41
New addres_space_operations methods are added:
int swapon(struct file *);
int swapoff(struct file *);
int swap_out(struct file *, struct page *, struct writeback_control *);
int swap_in(struct file *, struct page *);
When during sys_swapon() the ->swapon() method is found and returns no error
the swapper_space.a_ops will proxy to sis->swap_file->f_mapping->a_ops, and
make use of ->swap_{out,in}() to write/read swapcache pages.
The ->swapon() method will be used to communicate to the file that the VM
relies on it, and the address_space should take adequate measures (like
reserving memory for mempools or the like). The ->swapoff() method will be
called on sys_swapoff() when ->swapon() was found and returned no error.
This new interface can be used to obviate the need for ->bmap in the swapfile
code. A filesystem would need to load (and maybe even allocate) the full block
map for a file into memory and pin it there on ->swapon() so that
->swap_{out,in}() have instant access to it. It can be released on ->swapoff().
The reason to provide ->swap_{out,in}() over using {write,read}page() is to
1) make a distinction between swapcache and pagecache pages, and
2) to provide a struct file * for credential context (normally not needed
in the context of writepage, as the page content is normally dirtied
using either of the following interfaces:
write_{begin,end}()
{prepare,commit}_write()
page_mkwrite()
which do have the file context.
[miklos@szeredi.hu: cleanups]
Signed-off-by: Peter Zijlstra <redacted>
---
Documentation/filesystems/Locking | 22 ++++++++++++++++
Documentation/filesystems/vfs.txt | 18 +++++++++++++
include/linux/buffer_head.h | 2 -
include/linux/fs.h | 9 ++++++
include/linux/swap.h | 4 ++
mm/page_io.c | 52 ++++++++++++++++++++++++++++++++++++++
mm/swap_state.c | 4 +-
mm/swapfile.c | 32 +++++++++++++++++++++--
8 files changed, 137 insertions(+), 6 deletions(-)
Index: linux-2.6/include/linux/swap.h
===================================================================
@@ -121,6 +121,7 @@ enum {SWP_USED=(1<<0),/* is slot in swap_info[] used? */SWP_WRITEOK=(1<<1),/* ok to write to this swap? */SWP_ACTIVE=(SWP_USED|SWP_WRITEOK),+SWP_FILE=(1<<2),/* file swap area *//* add others here before... */SWP_SCANNING=(1<<8),/* refcount in scan_swap_map */};
@@ -169,6 +169,10 @@ prototypes: int (*direct_IO)(int, struct kiocb *, const struct iovec *iov, loff_t offset, unsigned long nr_segs); int (*launder_page) (struct page *);+ int (*swapon) (struct file *);+ int (*swapoff) (struct file *);+ int (*swap_out) (struct file *, struct page *, struct writeback_control *);+ int (*swap_in) (struct file *, struct page *); locking rules: All except set_page_dirty may block
@@ -190,6 +194,10 @@ invalidatepage: no yes releasepage: no yes direct_IO: no launder_page: no yes+swapon no+swapoff no+swap_out no yes, unlocks+swap_in no yes, unlocks ->prepare_write(), ->commit_write(), ->sync_page() and ->readpage() may be called from the request handler (/dev/loop).
@@ -289,6 +297,20 @@ cleaned, or an error value if not. Note getting mapped back in and redirtied, it needs to be kept locked across the entire operation.+ ->swapon() will be called with a non-zero argument on files backing+(non block device backed) swapfiles. A return value of zero indicates success,+in which case this file can be used for backing swapspace. The swapspace+operations will be proxied to the address space operations.++ ->swapoff() will be called in the sys_swapoff() path when ->swapon()+returned success.++ ->swap_out() when swapon() returned success, this method is used to+write the swap page.++ ->swap_in() when swapon() returned success, this method is used to+read the swap page.+ Note: currently almost all instances of address_space methods are using BKL for internal serialization and that's one of the worst sources of contention. Normally they are calling library functions (in fs/buffer.c)
@@ -539,6 +539,11 @@ struct address_space_operations { /* migrate the contents of a page to the specified target */ int (*migratepage) (struct page *, struct page *); int (*launder_page) (struct page *);+ int (*swapon)(struct file *);+ int (*swapoff)(struct file *);+ int (*swap_out)(struct file *file, struct page *page,+ struct writeback_control *wbc);+ int (*swap_in)(struct file *file, struct page *page); }; writepage: called by the VM to write a dirty page to backing store.
@@ -724,6 +729,19 @@ struct address_space_operations { prevent redirtying the page, it is kept locked during the whole operation.+ swapon: Called when swapon is used on a file. A+ return value of zero indicates success, in which case this+ file can be used to back swapspace. The swapspace operations+ will be proxied to this address space's ->swap_{out,in} methods.++ swapoff: Called during swapoff on files where swapon was successfull.++ swap_out: Called to write a swapcache page to a backing store, similar to+ writepage.++ swap_in: Called to read a swapcache page from a backing store, similar to+ readpage.+ The File Object ===============
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:53:57
Failing to allocate a cache entry will only harm performance not correctness.
Do not consume valuable reserve pages for something like that.
Signed-off-by: Peter Zijlstra <redacted>
Acked-by: James Morris <jmorris@namei.org>
---
security/selinux/avc.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
Index: linux-2.6/security/selinux/avc.c
===================================================================
@@ -127,13 +127,13 @@ static struct nfs_page *nfs_page_find_re/* Adjust the file length if we're writing beyond the end */staticvoidnfs_grow_file(structpage*page,unsignedintoffset,unsignedintcount){-structinode*inode=page->mapping->host;+structinode*inode=page_file_mapping(page)->host;loff_tend,i_size=i_size_read(inode);pgoff_tend_index=(i_size-1)>>PAGE_CACHE_SHIFT;-if(i_size>0&&page->index<end_index)+if(i_size>0&&page_file_index(page)<end_index)return;-end=((loff_t)page->index<<PAGE_CACHE_SHIFT)+((loff_t)offset+count);+end=page_file_offset(page)+((loff_t)offset+count);if(i_size>=end)return;nfs_inc_stats(inode,NFSIOS_EXTENDWRITE);
@@ -146,7 +146,7 @@ static void nfs_grow_file(struct page *pstaticvoidnfs_set_pageerror(structpage*page){SetPageError(page);-nfs_zap_mapping(page->mapping->host,page->mapping);+nfs_zap_mapping(page_file_mapping(page)->host,page_file_mapping(page));}/* We can set the PG_uptodate flag if we see that a write request
@@ -187,7 +187,7 @@ static int nfs_set_page_writeback(structintret=test_set_page_writeback(page);if(!ret){-structinode*inode=page->mapping->host;+structinode*inode=page_file_mapping(page)->host;structnfs_server*nfss=NFS_SERVER(inode);if(atomic_long_inc_return(&nfss->writeback)>
@@ -199,7 +199,7 @@ static int nfs_set_page_writeback(structstaticvoidnfs_end_page_writeback(structpage*page){-structinode*inode=page->mapping->host;+structinode*inode=page_file_mapping(page)->host;structnfs_server*nfss=NFS_SERVER(inode);end_page_writeback(page);
@@ -686,7 +687,7 @@ int nfs_flush_incompatible(struct file *nfs_release_request(req);if(!do_flush)return0;-status=nfs_wb_page(page->mapping->host,page);+status=nfs_wb_page(page_file_mapping(page)->host,page);}while(status==0);returnstatus;}
@@ -712,7 +713,7 @@ int nfs_updatepage(struct file *file, stunsignedintoffset,unsignedintcount){structnfs_open_context*ctx=nfs_file_open_context(file);-structinode*inode=page->mapping->host;+structinode*inode=page_file_mapping(page)->host;intstatus=0;nfs_inc_stats(inode,NFSIOS_VFSUPDATEPAGE);
@@ -720,7 +721,7 @@ int nfs_updatepage(struct file *file, stdprintk("NFS: nfs_updatepage(%s/%s %d@%lld)\n",file->f_path.dentry->d_parent->d_name.name,file->f_path.dentry->d_name.name,count,-(longlong)(page_offset(page)+offset));+(longlong)(page_file_offset(page)+offset));/* If we're not using byte range locks, and we know the page*isuptodate,itmaybemoreefficienttoextendthewrite
From: Nick Piggin <hidden> Date: 2008-07-24 14:54:41
On Friday 25 July 2008 00:01, Peter Zijlstra wrote:
With the introduction of the shared dirty page accounting in .19, NFS
should not be able to surpise the VM with all dirty pages. Thus it should
always be able to free some memory. Hence no more need for mempools.
It seems like a very backward step to me to go from a hard guarantee
to some heuristic that could break for someone in some particular
setup.
Filling with dirty pages isn't the only way to exaust free reclaimable
memory remember, it can also happen maybe due to mlock or kernel
allocations.
Is there pressing reason to remove them?
From: Peter Zijlstra <hidden> Date: 2008-07-24 14:55:23
On Fri, 2008-07-25 at 00:46 +1000, Nick Piggin wrote:
On Friday 25 July 2008 00:01, Peter Zijlstra wrote:
quoted
With the introduction of the shared dirty page accounting in .19, NFS
should not be able to surpise the VM with all dirty pages. Thus it should
always be able to free some memory. Hence no more need for mempools.
It seems like a very backward step to me to go from a hard guarantee
to some heuristic that could break for someone in some particular
setup.
Filling with dirty pages isn't the only way to exaust free reclaimable
memory remember, it can also happen maybe due to mlock or kernel
allocations.
Is there pressing reason to remove them?
There was some funny interaction between mempools and PF_MEMALLOC as
used to write out swap pages iirc.
Specifially, the mempool would strip it and not provide memory even
though we needed it.
I've never hit a deadlock due to this patch in my testing, although
admittedly there might be a hole in my testing.
I can either 'fix' mempools, or replace it with the reserve framework
introduced earlier in this patch-set.
__GFP_MEMALLOC will allow the allocation to disregard the watermarks,
much like PF_MEMALLOC.
It allows one to pass along the memalloc state in object related allocation
flags as opposed to task related flags, such as sk->sk_allocation.
Is this properly name?
page alloc is always "mem alloc".
you wrote comment as "Use emergency reserves" and
this flag works to turn on ALLOC_NO_WATERMARKS.
then, __GFP_NO_WATERMARK or __GFP_EMERGENCY are better?
From: Peter Zijlstra <hidden> Date: 2008-07-25 09:35:40
On Fri, 2008-07-25 at 18:29 +0900, KOSAKI Motohiro wrote:
Hi Peter,
quoted
__GFP_MEMALLOC will allow the allocation to disregard the watermarks,
much like PF_MEMALLOC.
It allows one to pass along the memalloc state in object related allocation
flags as opposed to task related flags, such as sk->sk_allocation.
Is this properly name?
page alloc is always "mem alloc".
you wrote comment as "Use emergency reserves" and
this flag works to turn on ALLOC_NO_WATERMARKS.
then, __GFP_NO_WATERMARK or __GFP_EMERGENCY are better?
We've been through this pick a better name thing several times :-/
Yes I agree, __GFP_MEMALLOC is a misnomer, however its consistent with
PF_MEMALLOC and __GFP_NOMEMALLOC - of which people know the semantics.
Creating a new name with similar semantics can only serve to confuse.
So unless enough people think its worth renaming all of them, I think
we're better off with this name.
On Fri, 2008-07-25 at 18:29 +0900, KOSAKI Motohiro wrote:
quoted
Hi Peter,
quoted
__GFP_MEMALLOC will allow the allocation to disregard the watermarks,
much like PF_MEMALLOC.
It allows one to pass along the memalloc state in object related allocation
flags as opposed to task related flags, such as sk->sk_allocation.
Is this properly name?
page alloc is always "mem alloc".
you wrote comment as "Use emergency reserves" and
this flag works to turn on ALLOC_NO_WATERMARKS.
then, __GFP_NO_WATERMARK or __GFP_EMERGENCY are better?
We've been through this pick a better name thing several times :-/
Yes I agree, __GFP_MEMALLOC is a misnomer, however its consistent with
PF_MEMALLOC and __GFP_NOMEMALLOC - of which people know the semantics.
Creating a new name with similar semantics can only serve to confuse.
Ah, understand.
Thanks.
Agreed to __GFP_MEMALLOC.
So unless enough people think its worth renaming all of them, I think
we're better off with this name.
From: Peter Zijlstra <hidden> Date: 2008-07-25 10:57:57
On Fri, 2008-07-25 at 19:46 +0900, KOSAKI Motohiro wrote:
quoted
GFP_NOFS is not enough, since swap traffic is IO, hence fall back to GFP_NOIO.
this comment imply turn on GFP_NOIO, but the code is s/NOFS/NOIO/. why?
Does the misunderstanding stem from the use of 'enough'?
GFP_NOFS is _more_ permissive than GFP_NOIO in that it will initiate IO,
just not of any filesystem data.
The problem is that previuosly NOFS was correct because that avoids
recursion into the NFS code, it now is not, because also IO (swap) can
lead to this recursion.
Therefore we must tighten the restriction and use NOIO.
From: Peter Zijlstra <hidden> Date: 2008-07-25 11:19:21
On Fri, 2008-07-25 at 20:15 +0900, KOSAKI Motohiro wrote:
quoted
On Fri, 2008-07-25 at 19:46 +0900, KOSAKI Motohiro wrote:
quoted
quoted
GFP_NOFS is not enough, since swap traffic is IO, hence fall back to GFP_NOIO.
this comment imply turn on GFP_NOIO, but the code is s/NOFS/NOIO/. why?
Does the misunderstanding stem from the use of 'enough'?
GFP_NOFS is _more_ permissive than GFP_NOIO in that it will initiate IO,
just not of any filesystem data.
quoted
The problem is that previuosly NOFS was correct because that avoids
recursion into the NFS code, it now is not, because also IO (swap) can
lead to this recursion.
Thanks nicer explain.
So, I hope add above 3 line to patch description.
On Fri, 2008-07-25 at 19:46 +0900, KOSAKI Motohiro wrote:
quoted
quoted
GFP_NOFS is not enough, since swap traffic is IO, hence fall back to GFP_NOIO.
this comment imply turn on GFP_NOIO, but the code is s/NOFS/NOIO/. why?
Does the misunderstanding stem from the use of 'enough'?
GFP_NOFS is _more_ permissive than GFP_NOIO in that it will initiate IO,
just not of any filesystem data.
The problem is that previuosly NOFS was correct because that avoids
recursion into the NFS code, it now is not, because also IO (swap) can
lead to this recursion.
Thanks nicer explain.
So, I hope add above 3 line to patch description.
Cheers!
@@ -163,11 +163,11 @@ static struct notifier_block slab_notifi#endifstaticenum{-DOWN,/* No slab functionality available */+DOWN=0,/* No slab functionality available */PARTIAL,/* kmem_cache_open() works but kmalloc does not */UP,/* Everything works but does not show up in sysfs */SYSFS/* Sysfs up */-}slab_state=DOWN;+}slab_state;/* A list of all slab caches on the system */staticDECLARE_RWSEM(slub_lock);
From: Pekka Enberg <hidden> Date: 2008-07-28 10:06:21
Hi Peter,
On Thu, 2008-07-24 at 16:00 +0200, Peter Zijlstra wrote:
quoted hunk
+/*+ * alloc wrappers+ */+
Hmm, I'm not sure I like the use of __kmalloc_track_caller() (even
though you do add the wrappers for SLUB). The functions really are SLAB
internals so I'd prefer to see kmalloc_reserve() moved to the
allocators.
quoted hunk
+void *___kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip,+ struct mem_reserve *res, int *emerg)+{
Why don't we discharge from the reserve here if !obj?
quoted hunk
+ if (emerg)+ *emerg |= 1;++out:+ return obj;+}++void __kfree_reserve(void *obj, struct mem_reserve *res, int emerg)
I don't see 'emerg' used anywhere.
quoted hunk
+{+ size_t size = ksize(obj);++ kfree(obj);
We're trying to get rid of kfree() so I'd __kfree_reserve() could to
mm/sl?b.c. Matt, thoughts?
quoted hunk
+ /*+ * ksize gives the full allocated size vs the requested size we used to+ * charge; however since we round up to the nearest power of two, this+ * should all work nicely.+ */+ mem_reserve_kmalloc_charge(res, -size);+}
From: Peter Zijlstra <hidden> Date: 2008-07-28 10:17:24
On Mon, 2008-07-28 at 13:06 +0300, Pekka Enberg wrote:
Hi Peter,
On Thu, 2008-07-24 at 16:00 +0200, Peter Zijlstra wrote:
quoted
+/*+ * alloc wrappers+ */+
Hmm, I'm not sure I like the use of __kmalloc_track_caller() (even
though you do add the wrappers for SLUB). The functions really are SLAB
internals so I'd prefer to see kmalloc_reserve() moved to the
allocators.
See below..
quoted
+void *___kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip,+ struct mem_reserve *res, int *emerg)+{
This function could use some comments...
Yes, my latest does have those.. let me paste the relevant bit:
+void *___kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip,
+ struct mem_reserve *res, int *emerg)
+{
+ void *obj;
+ gfp_t gfp;
+
+ /*
+ * Try a regular allocation, when that fails and we're not entitled
+ * to the reserves, fail.
+ */
+ gfp = flags | __GFP_NOMEMALLOC | __GFP_NOWARN;
+ obj = __kmalloc_node_track_caller(size, gfp, node, ip);
+
+ if (obj || !(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS))
+ goto out;
+
+ /*
+ * If we were given a reserve to charge against, try that.
+ */
+ if (res && !mem_reserve_kmalloc_charge(res, size)) {
+ /*
+ * If we failed to charge and we're not allowed to wait for
+ * it to succeed, bail.
+ */
+ if (!(flags & __GFP_WAIT))
+ goto out;
+
+ /*
+ * Wait for a successfull charge against the reserve. All
+ * uncharge operations against this reserve will wake us up.
+ */
+ wait_event(res->waitqueue,
+ mem_reserve_kmalloc_charge(res, size));
+
+ /*
+ * After waiting for it, again try a regular allocation.
+ * Pressure could have lifted during our sleep. If this
+ * succeeds, uncharge the reserve.
+ */
+ obj = __kmalloc_node_track_caller(size, gfp, node, ip);
+ if (obj) {
+ mem_reserve_kmalloc_charge(res, -size);
+ goto out;
+ }
+ }
+
+ /*
+ * Regular allocation failed, and we've successfully charged our
+ * requested usage against the reserve. Do the emergency allocation.
+ */
+ obj = __kmalloc_node_track_caller(size, flags, node, ip);
+ WARN_ON(!obj);
+ if (emerg)
+ *emerg |= 1;
+
+out:
+ return obj;
+}
We're trying to get rid of kfree() so I'd __kfree_reserve() could to
mm/sl?b.c. Matt, thoughts?
My issue with moving these helpers into mm/sl?b.c is that it would
require replicating all this code 3 times. Even though the functionality
is (or should) be invariant to the actual slab implementation.
quoted
+ /*+ * ksize gives the full allocated size vs the requested size we used to+ * charge; however since we round up to the nearest power of two, this+ * should all work nicely.+ */+ mem_reserve_kmalloc_charge(res, -size);+}
@@ -163,11 +163,11 @@ static struct notifier_block slab_notifi#endifstaticenum{-DOWN,/* No slab functionality available */+DOWN=0,/* No slab functionality available */PARTIAL,/* kmem_cache_open() works but kmalloc does not */UP,/* Everything works but does not show up in sysfs */SYSFS/* Sysfs up */-}slab_state=DOWN;+}slab_state;/* A list of all slab caches on the system */staticDECLARE_RWSEM(slub_lock);
From: Pekka Enberg <hidden> Date: 2008-07-28 10:30:14
Hi Peter,
On Mon, 2008-07-28 at 12:17 +0200, Peter Zijlstra wrote:
On Mon, 2008-07-28 at 13:06 +0300, Pekka Enberg wrote:
quoted
Hi Peter,
On Thu, 2008-07-24 at 16:00 +0200, Peter Zijlstra wrote:
quoted
+/*+ * alloc wrappers+ */+
Hmm, I'm not sure I like the use of __kmalloc_track_caller() (even
though you do add the wrappers for SLUB). The functions really are SLAB
internals so I'd prefer to see kmalloc_reserve() moved to the
allocators.
See below..
quoted
quoted
+void *___kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip,+ struct mem_reserve *res, int *emerg)+{
This function could use some comments...
Yes, my latest does have those.. let me paste the relevant bit:
+void *___kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip,
+ struct mem_reserve *res, int *emerg)
+{
+ void *obj;
+ gfp_t gfp;
+
+ /*
+ * Try a regular allocation, when that fails and we're not entitled
+ * to the reserves, fail.
+ */
+ gfp = flags | __GFP_NOMEMALLOC | __GFP_NOWARN;
+ obj = __kmalloc_node_track_caller(size, gfp, node, ip);
+
+ if (obj || !(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS))
+ goto out;
+
+ /*
+ * If we were given a reserve to charge against, try that.
+ */
+ if (res && !mem_reserve_kmalloc_charge(res, size)) {
+ /*
+ * If we failed to charge and we're not allowed to wait for
+ * it to succeed, bail.
+ */
+ if (!(flags & __GFP_WAIT))
+ goto out;
+
+ /*
+ * Wait for a successfull charge against the reserve. All
+ * uncharge operations against this reserve will wake us up.
+ */
+ wait_event(res->waitqueue,
+ mem_reserve_kmalloc_charge(res, size));
+
+ /*
+ * After waiting for it, again try a regular allocation.
+ * Pressure could have lifted during our sleep. If this
+ * succeeds, uncharge the reserve.
+ */
+ obj = __kmalloc_node_track_caller(size, gfp, node, ip);
+ if (obj) {
+ mem_reserve_kmalloc_charge(res, -size);
+ goto out;
+ }
+ }
+
+ /*
+ * Regular allocation failed, and we've successfully charged our
+ * requested usage against the reserve. Do the emergency allocation.
+ */
+ obj = __kmalloc_node_track_caller(size, flags, node, ip);
+ WARN_ON(!obj);
+ if (emerg)
+ *emerg |= 1;
+
+out:
+ return obj;
+}
We're trying to get rid of kfree() so I'd __kfree_reserve() could to
mm/sl?b.c. Matt, thoughts?
My issue with moving these helpers into mm/sl?b.c is that it would
require replicating all this code 3 times. Even though the functionality
is (or should) be invariant to the actual slab implementation.
Right, I guess we could just rename ksize() to something else then and
keep it internal to mm/.
quoted
quoted
+ /*+ * ksize gives the full allocated size vs the requested size we used to+ * charge; however since we round up to the nearest power of two, this+ * should all work nicely.+ */+ mem_reserve_kmalloc_charge(res, -size);+}
Why don't we discharge from the reserve here if !obj?
Well, this allocation should never fail:
- we reserved memory
- we accounted/throttle its usage
Thus this allocation should always succeed.
But if it *does* fail, it doesn't help that we mess up the reservation
counts, no?
I guess you're right there. Will fix. Thanks!
quoted
quoted
quoted
+{+ size_t size = ksize(obj);++ kfree(obj);
We're trying to get rid of kfree() so I'd __kfree_reserve() could to
mm/sl?b.c. Matt, thoughts?
My issue with moving these helpers into mm/sl?b.c is that it would
require replicating all this code 3 times. Even though the functionality
is (or should) be invariant to the actual slab implementation.
Right, I guess we could just rename ksize() to something else then and
keep it internal to mm/.
That would be nice - we can stuff it into mm/internal.h or somesuch.
Also, you might have noticed, I still need to do everything SLOB. The
last time I rewrote all this code I was still hoping Linux would 'soon'
have a single slab allocator, but evidently we're still going with 3 for
now.. :-/
So I guess I can no longer hide behind that and will have to bite the
bullet and write the SLOB bits..
quoted
quoted
quoted
+ /*+ * ksize gives the full allocated size vs the requested size we used to+ * charge; however since we round up to the nearest power of two, this+ * should all work nicely.+ */+ mem_reserve_kmalloc_charge(res, -size);+}
From: Pekka Enberg <hidden> Date: 2008-07-28 10:41:37
Hi Peter,
On Mon, 2008-07-28 at 12:39 +0200, Peter Zijlstra wrote:
Also, you might have noticed, I still need to do everything SLOB. The
last time I rewrote all this code I was still hoping Linux would 'soon'
have a single slab allocator, but evidently we're still going with 3 for
now.. :-/
So I guess I can no longer hide behind that and will have to bite the
bullet and write the SLOB bits..
Oh, I don't expect SLOB to go away anytime soon. We are still trying to
get rid of SLAB, though, but there are some TPC regressions that we
don't have a reproducible test case for so that effort has stalled a
bit.
Pekka
From: Matt Mackall <hidden> Date: 2008-07-28 16:53:08
On Mon, 2008-07-28 at 13:06 +0300, Pekka Enberg wrote:
We're trying to get rid of kfree() so I'd __kfree_reserve() could to
mm/sl?b.c. Matt, thoughts?
I think you mean ksize there. My big issue is that we need to make it
clear that ksize pairs -only- with kmalloc and that
ksize(kmem_cache_alloc(...)) is a categorical error. Preferably, we do
this by giving it a distinct name, like kmalloc_size(). We can stick an
underbar in front of it to suggest you ought not be using it too.
quoted
+ /*
+ * ksize gives the full allocated size vs the requested size we
used to
quoted
+ * charge; however since we round up to the nearest power of two,
this
quoted
+ * should all work nicely.
+ */
SLOB doesn't do this, of course. But does that matter? I think you want
to charge the actual allocation size to the reserve in all cases, no?
That probably means calling ksize() on both alloc and free.
--
Mathematics is the supreme nostalgia of our time.
From: Matt Mackall <hidden> Date: 2008-07-28 17:03:43
On Mon, 2008-07-28 at 12:39 +0200, Peter Zijlstra wrote:
Also, you might have noticed, I still need to do everything SLOB. The
last time I rewrote all this code I was still hoping Linux would 'soon'
have a single slab allocator, but evidently we're still going with 3 for
now.. :-/
So I guess I can no longer hide behind that and will have to bite the
bullet and write the SLOB bits..
I haven't seen the rest of this thread, but I presume this is part of
your OOM-avoidance for network I/O framework?
SLOB can be pretty easily expanded to handle a notion of independent
allocation arenas as there are only a couple global variables to switch
between. kfree will also return allocations to the page list (and
therefore arena) from whence they came. That may make it pretty simple
to create and prepopulate reserve pools.
--
Mathematics is the supreme nostalgia of our time.
From: Peter Zijlstra <hidden> Date: 2008-07-28 17:14:11
On Mon, 2008-07-28 at 11:49 -0500, Matt Mackall wrote:
On Mon, 2008-07-28 at 13:06 +0300, Pekka Enberg wrote:
quoted
We're trying to get rid of kfree() so I'd __kfree_reserve() could to
mm/sl?b.c. Matt, thoughts?
I think you mean ksize there. My big issue is that we need to make it
clear that ksize pairs -only- with kmalloc and that
ksize(kmem_cache_alloc(...)) is a categorical error. Preferably, we do
this by giving it a distinct name, like kmalloc_size(). We can stick an
underbar in front of it to suggest you ought not be using it too.
Right, both make sense, so _kmalloc_size() has my vote.
quoted
quoted
+ /*
+ * ksize gives the full allocated size vs the requested size we
used to
quoted
+ * charge; however since we round up to the nearest power of two,
this
quoted
+ * should all work nicely.
+ */
SLOB doesn't do this, of course. But does that matter? I think you want
to charge the actual allocation size to the reserve in all cases, no?
That probably means calling ksize() on both alloc and free.
Like said, I still need to do all the SLOB reservation stuff. That
includes coming up with upper bound fragmentation loss.
For SL[UA]B I use roundup_power_of_two for kmalloc sizes. Thus with the
above ksize(), if we did p=kmalloc(x), then we'd account
roundup_power_of_two(x), and that should be equal to
roundup_power_of_two(ksize(p)), as ksize will always be smaller or equal
to the roundup.
I'm guessing the power of two upper bound is good for SLOB too -
although I haven't tried proving it wrong or tighetening it.
Only the kmem_cache_* reservation stuff would need some extra attention
with SLOB.
From: Peter Zijlstra <hidden> Date: 2008-07-28 17:14:42
On Mon, 2008-07-28 at 11:59 -0500, Matt Mackall wrote:
On Mon, 2008-07-28 at 12:39 +0200, Peter Zijlstra wrote:
quoted
Also, you might have noticed, I still need to do everything SLOB. The
last time I rewrote all this code I was still hoping Linux would 'soon'
have a single slab allocator, but evidently we're still going with 3 for
now.. :-/
So I guess I can no longer hide behind that and will have to bite the
bullet and write the SLOB bits..
I haven't seen the rest of this thread, but I presume this is part of
your OOM-avoidance for network I/O framework?
Yes indeed.
SLOB can be pretty easily expanded to handle a notion of independent
allocation arenas as there are only a couple global variables to switch
between. kfree will also return allocations to the page list (and
therefore arena) from whence they came. That may make it pretty simple
to create and prepopulate reserve pools.
Right - currently we let all the reserves sit on the free page list. The
advantage there is that it also helps the anti-frag stuff, due to having
larger free lists.
From: Pekka Enberg <hidden> Date: 2008-07-30 12:21:55
Hi Peter,
On Thu, 2008-07-24 at 16:00 +0200, Peter Zijlstra wrote:
quoted hunk
plain text document attachment (mm-kmem_estimate_pages.patch)
Provide a method to get the upper bound on the pages needed to allocate
a given number of objects from a given kmem_cache.
This lays the foundation for a generic reserve framework as presented in
a later patch in this series. This framework needs to convert object demand
(kmalloc() bytes, kmem_cache_alloc() objects) to pages.
Signed-off-by: Peter Zijlstra <redacted>
---
include/linux/slab.h | 4 ++
mm/slab.c | 75 +++++++++++++++++++++++++++++++++++++++++++
mm/slub.c | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 166 insertions(+)
Index: linux-2.6/include/linux/slab.h
===================================================================
/*
* Allocator specific definitions. These are mainly used to establish optimized
Index: linux-2.6/mm/slub.c
===================================================================
Hmm, I'm not sure why slab with just one object is treated separately
here. Surely you have per-CPU slabs then as well?
quoted hunk
+ /*+ * Account the possible additional overhead if per cpu slabs+ * are currently empty and have to be allocated. This is very+ * unlikely but a possible scenario immediately after+ * kmem_cache_shrink.+ */+ pages += num_online_cpus() << oo_order(x);
Isn't this problematic with CPU hotplug? Shouldn't we use
num_possible_cpus() here?
@@ -2789,6 +2825,57 @@ void kfree(const void *x) EXPORT_SYMBOL(kfree); /*+ * Calculate the upper bound of pages required to sequentially allocate+ * @count objects of @size bytes from kmalloc given @flags.+ */+unsigned kmalloc_estimate_fixed(size_t size, gfp_t flags, int count)+{+ struct kmem_cache *s = get_slab(size, flags);+ if (!s)+ return 0;++ return kmem_alloc_estimate(s, flags, count);++}+EXPORT_SYMBOL_GPL(kmalloc_estimate_fixed);++/*+ * Calculate the upper bound of pages requires to sequentially allocate @bytes+ * from kmalloc in an unspecified number of allocations of nonuniform size.+ */+unsigned kmalloc_estimate_variable(gfp_t flags, size_t bytes)+{+ int i;+ unsigned long pages;++ /*+ * multiply by two, in order to account the worst case slack space+ * due to the power-of-two allocation sizes.+ */+ pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE);
For bytes > PAGE_SIZE this doesn't look right (for SLUB). We do page
allocator pass-through which means that we'll be grabbing high order
pages which can be bigger than what 'pages' is here.
quoted hunk
+
+ /*
+ * add the kmem_cache overhead of each possible kmalloc cache
+ */
+ for (i = 1; i < PAGE_SHIFT; i++) {
+ struct kmem_cache *s;
+
+#ifdef CONFIG_ZONE_DMA
+ if (unlikely(flags & SLUB_DMA))
+ s = dma_kmalloc_cache(i, flags);
+ else
+#endif
+ s = &kmalloc_caches[i];
+
+ if (s)
+ pages += kmem_alloc_estimate(s, flags, 0);
+ }
+
+ return pages;
+}
+EXPORT_SYMBOL_GPL(kmalloc_estimate_variable);
+
+/*
* kmem_cache_shrink removes empty slabs from the partial lists and sorts
* the remaining slabs by the number of items in use. The slabs with the
* most items in use come first. New allocations will then fill those up
Index: linux-2.6/mm/slab.c
===================================================================
Heh, yes please. Or add a comment why it doesn't matter.
quoted hunk
+ */+ nr_pages += 1 + ilog2(nr_pages);++ return nr_pages;+}++/*+ * Calculate the upper bound of pages required to sequentially allocate+ * @count objects of @size bytes from kmalloc given @flags.+ */+unsigned kmalloc_estimate_fixed(size_t size, gfp_t flags, int count)+{+ struct kmem_cache *s = kmem_find_general_cachep(size, flags);+ if (!s)+ return 0;++ return kmem_alloc_estimate(s, flags, count);+}+EXPORT_SYMBOL_GPL(kmalloc_estimate_fixed);++/*+ * Calculate the upper bound of pages requires to sequentially allocate @bytes+ * from kmalloc in an unspecified number of allocations of nonuniform size.+ */+unsigned kmalloc_estimate_variable(gfp_t flags, size_t bytes)+{+ unsigned long pages;+ struct cache_sizes *csizep = malloc_sizes;++ /*+ * multiply by two, in order to account the worst case slack space+ * due to the power-of-two allocation sizes.+ */+ pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE);++ /*+ * add the kmem_cache overhead of each possible kmalloc cache+ */+ for (csizep = malloc_sizes; csizep->cs_cachep; csizep++) {+ struct kmem_cache *s;++#ifdef CONFIG_ZONE_DMA+ if (unlikely(flags & __GFP_DMA))+ s = csizep->cs_dmacachep;+ else+#endif+ s = csizep->cs_cachep;++ if (s)+ pages += kmem_alloc_estimate(s, flags, 0);+ }++ return pages;+}+EXPORT_SYMBOL_GPL(kmalloc_estimate_variable);++/* * This initializes kmem_list3 or resizes various caches for all nodes. */ static int alloc_kmemlist(struct kmem_cache *cachep)
From: Pekka Enberg <hidden> Date: 2008-07-30 12:36:16
On Thu, 2008-07-24 at 16:00 +0200, Peter Zijlstra wrote:
plain text document attachment (mm-setup_per_zone_pages_min.patch)
There is a small race between the procfs caller and the memory hotplug caller
of setup_per_zone_pages_min(). Not a big deal, but the next patch will add yet
another caller. Time to close the gap.
Signed-off-by: Peter Zijlstra <redacted>
Looks good to me.
Reviewed-by: Pekka Enberg <redacted>
Sounds good, I'll do some sed magic on the patch-set to make it happen.
quoted
/*
* Allocator specific definitions. These are mainly used to establish optimized
Index: linux-2.6/mm/slub.c
===================================================================
Hmm, I'm not sure why slab with just one object is treated separately
here. Surely you have per-CPU slabs then as well?
The thought was that if the slab only contains 1 obj, then the per-cpu
slabs are always full (or empty but already there), so you don't loose
memory to other cpu's having half-filled slabs.
Say you want to reserve memory for 10 object.
In the 1 object per slab case, you will always allocate a slab, no
matter what cpu you do the allocation on.
With say, 16 objects per slab and allocations spread across 2 cpus, you
have to allow for per-cpu slabs to be half-filled.
quoted
+ /*+ * Account the possible additional overhead if per cpu slabs+ * are currently empty and have to be allocated. This is very+ * unlikely but a possible scenario immediately after+ * kmem_cache_shrink.+ */+ pages += num_online_cpus() << oo_order(x);
Isn't this problematic with CPU hotplug? Shouldn't we use
num_possible_cpus() here?
ACK, thanks!
quoted
+/*+ * Calculate the upper bound of pages requires to sequentially allocate @bytes+ * from kmalloc in an unspecified number of allocations of nonuniform size.+ */+unsigned kmalloc_estimate_variable(gfp_t flags, size_t bytes)+{+ int i;+ unsigned long pages;++ /*+ * multiply by two, in order to account the worst case slack space+ * due to the power-of-two allocation sizes.+ */+ pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE);
For bytes > PAGE_SIZE this doesn't look right (for SLUB). We do page
allocator pass-through which means that we'll be grabbing high order
pages which can be bigger than what 'pages' is here.
Hehe - you actually made me think here.
Satisfying allocations from a bucket distribution with power-of-two
(which page alloc order satisfies) has a worst case slack space of:
S(x) = 2^n - (2^(n-1)) - 1, n = ceil(log2(x))
This can be seen for the cases where x = 2^i + 1.
If we approximate S(x) by 2^(n-1) and compute the slack ratio for any
given x:
R(x) ~ 2^n / 2^(n-1) = 2
We'll see that for any amount of x, we can only use half that due to
slack space.
Therefore, by multiplying the demand @bytes by 2 we'll always have
enough to cover the worst case slack considering the power-of-two
allocation buckets.
In example, if @bytes asks for 4 pages + 1 byte = 16385 bytes (assuming
4k pages), then the above will request 8 pages + 2 bytes, rounded up to
pages, is 9 pages. Which is enough to satisfy the order 3 allocation
needed for the 8 contiguous pages to store the requested 16385 bytes.
Heh, yes please. Or add a comment why it doesn't matter.
Since you were the one I cribbed that comment from some (long) time ago,
can you advise on how well the below approximation is to an upper bound
on the above factors - assuming SLAB will live long enough to make it
worth the effort?
Hmmm..... node? Maybe use the struct name? kmem_cache_node?
quoted
quoted
@@ -163,11 +163,11 @@ static struct notifier_block slab_notifi #endif static enum {- DOWN, /* No slab functionality available */+ DOWN = 0, /* No slab functionality available */ PARTIAL, /* kmem_cache_open() works but kmalloc does not */ UP, /* Everything works but does not show up in sysfs */ SYSFS /* Sysfs up */-} slab_state = DOWN;+} slab_state; /* A list of all slab caches on the system */ static DECLARE_RWSEM(slub_lock);
It defaults to the first enum value. We also do not initialize statics with zero.
Another width limitation that will limit the number of objects in a slab to 64k.
Also gcc does not get the fields, wont be able to optimize this as well and it will emit conversion from 16 bit loads.
Hmmm. You are moving it back on one line and Andrew will cut it up again later? This seems to be oscillating...
quoted
quoted
@@ -1799,11 +1796,11 @@ static int slub_nomerge; * slub_max_order specifies the order where we begin to stop considering the * number of objects in a slab as critical. If we reach slub_max_order then * we try to keep the page order as low as possible. So we accept more waste- * of space in favor of a small page order.+ * of space in favour of a small page order. * * Higher order allocations also allow the placement of more objects in a * slab and thereby reduce object handling overhead. If the user has- * requested a higher mininum order then we start with that one instead of+ * requested a higher minimum order then we start with that one instead of * the smallest order which will fit the object. */ static inline int slab_order(int size, int min_objects,
Hmmm..... node? Maybe use the struct name? kmem_cache_node?
quoted
quoted
quoted
@@ -163,11 +163,11 @@ static struct notifier_block slab_notifi #endif static enum {- DOWN, /* No slab functionality available */+ DOWN = 0, /* No slab functionality available */ PARTIAL, /* kmem_cache_open() works but kmalloc does not */ UP, /* Everything works but does not show up in sysfs */ SYSFS /* Sysfs up */-} slab_state = DOWN;+} slab_state; /* A list of all slab caches on the system */ static DECLARE_RWSEM(slub_lock);
It defaults to the first enum value. We also do not initialize statics with zero.
I seem to recall differently, I thought we explicitly init global stuff
to 0.
Another width limitation that will limit the number of objects in a slab to 64k.
Also gcc does not get the fields, wont be able to optimize this as well and it will emit conversion from 16 bit loads.
Well, it was already 16 by virtue of the old code.
GCC emmiting rubbish code due to the union would be sad - I can respin
without this.
Hmmm. You are moving it back on one line and Andrew will cut it up again later? This seems to be oscillating...
Gah, somehow I got convinced the result was <= 80..
quoted
quoted
quoted
@@ -1799,11 +1796,11 @@ static int slub_nomerge; * slub_max_order specifies the order where we begin to stop considering the * number of objects in a slab as critical. If we reach slub_max_order then * we try to keep the page order as low as possible. So we accept more waste- * of space in favor of a small page order.+ * of space in favour of a small page order. * * Higher order allocations also allow the placement of more objects in a * slab and thereby reduce object handling overhead. If the user has- * requested a higher mininum order then we start with that one instead of+ * requested a higher minimum order then we start with that one instead of * the smallest order which will fit the object. */ static inline int slab_order(int size, int min_objects,
From: Christoph Lameter <hidden> Date: 2008-07-30 20:03:23
Peter Zijlstra wrote:
quoted
quoted
+/*+ * Calculate the upper bound of pages requires to sequentially allocate @bytes+ * from kmalloc in an unspecified number of allocations of nonuniform size.+ */+unsigned kmalloc_estimate_variable(gfp_t flags, size_t bytes)+{+ int i;+ unsigned long pages;++ /*+ * multiply by two, in order to account the worst case slack space+ * due to the power-of-two allocation sizes.+ */+ pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE);
For bytes > PAGE_SIZE this doesn't look right (for SLUB). We do page
allocator pass-through which means that we'll be grabbing high order
pages which can be bigger than what 'pages' is here.
Satisfying allocations from a bucket distribution with power-of-two
(which page alloc order satisfies) has a worst case slack space of:
S(x) = 2^n - (2^(n-1)) - 1, n = ceil(log2(x))
This can be seen for the cases where x = 2^i + 1.
The needed bytes for a kmalloc allocation with size > PAGE_SIZE is
get_order(size) << PAGE_SHIFT bytes.
See kmalloc_large().
From: Peter Zijlstra <hidden> Date: 2008-09-30 12:41:47
On Thu, 2008-07-24 at 16:00 +0200, Peter Zijlstra wrote:
Latest version of the swap over nfs work.
Patches are against: v2.6.26-rc8-mm1
I still need to write some more comments in the reservation code.
Pekka, it uses ksize(), please have a look.
This version also deals with network namespaces.
Two things where I could do with some suggestsion:
- currently the sysctl code uses current->nrproxy.net_ns to obtain
the current network namespace
- the ipv6 route cache code has some initialization order issues
Daniel, have you ever found time to look at my namespace issues?
From: Daniel Lezcano <hidden> Date: 2008-09-30 15:46:54
Peter Zijlstra wrote:
On Thu, 2008-07-24 at 16:00 +0200, Peter Zijlstra wrote:
quoted
Latest version of the swap over nfs work.
Patches are against: v2.6.26-rc8-mm1
I still need to write some more comments in the reservation code.
Pekka, it uses ksize(), please have a look.
This version also deals with network namespaces.
Two things where I could do with some suggestsion:
- currently the sysctl code uses current->nrproxy.net_ns to obtain
the current network namespace
- the ipv6 route cache code has some initialization order issues
Daniel, have you ever found time to look at my namespace issues?
Oops, no. I was busy and I forgot, sorry.
Let me review the sysctl vs namespace part.
Thanks
-- Daniel
From: Daniel Lezcano <hidden> Date: 2008-10-01 11:38:48
Add reserves for INET.
The two big users seem to be the route cache and ip-fragment cache.
Reserve the route cache under generic RX reserve, its usage is bounded by
the high reclaim watermark, and thus does not need further accounting.
Reserve the ip-fragement caches under SKB data reserve, these add to the
SKB RX limit. By ensuring we can at least receive as much data as fits in
the reassmbly line we avoid fragment attack deadlocks.
Adds to the reserve tree:
total network reserve network TX reserve protocol TX pages network RX reserve + IPv6 route cache + IPv4 route cache SKB data reserve + IPv6 fragment cache + IPv4 fragment cache
Signed-off-by: Peter Zijlstra <redacted>
---
include/net/inet_frag.h | 7 +++
include/net/netns/ipv6.h | 4 ++
net/ipv4/inet_fragment.c | 3 +
net/ipv4/ip_fragment.c | 89 +++++++++++++++++++++++++++++++++++++++++++++--
net/ipv4/route.c | 72 +++++++++++++++++++++++++++++++++++++-
net/ipv6/af_inet6.c | 20 +++++++++-
net/ipv6/reassembly.c | 88 +++++++++++++++++++++++++++++++++++++++++++++-
net/ipv6/route.c | 66 ++++++++++++++++++++++++++++++++++
8 files changed, 341 insertions(+), 8 deletions(-)
Sorry for the delay ...
[ cut ]
I removed a big portion of code because the remarks below apply to the rest of the code.
quoted hunk
+static int sysctl_intvec_route(struct ctl_table *table,+ int __user *name, int nlen,+ void __user *oldval, size_t __user *oldlenp,+ void __user *newval, size_t newlen)+{+ struct net *net = current->nsproxy->net_ns;
I think you can use the container_of and get rid of using current->nsproxy->net_ns.
struct net *net = container_of(table->data, struct net,
ipv6.sysctl.ip6_rt_max_size);
Another solution could be to pass directly the sysctl structure pointer in the table data instead of
".data = &init_net.ipv6.sysctl.ip6_rt_max_size" when initializing the sysctl table below. But you have to set the right field value yourself.
quoted hunk
+ int write = (newval && newlen);+ int new_size, ret;++ mutex_lock(&net->ipv6.sysctl.ip6_rt_lock);++ if (write)+ table->data = &new_size;++ ret = sysctl_intvec(table, name, nlen, oldval, oldlenp, newval, newlen);++ if (!ret && write) {+ ret = mem_reserve_kmem_cache_set(&net->ipv6.ip6_rt_reserve,+ net->ipv6.ip6_dst_ops.kmem_cachep, new_size);+ if (!ret)+ net->ipv6.sysctl.ip6_rt_max_size = new_size;+ }++ if (write)+ table->data = &net->ipv6.sysctl.ip6_rt_max_size;++ mutex_unlock(&net->ipv6.sysctl.ip6_rt_lock);++ return ret;+}
Dancing with the table->data looks safe but it is not very nice.
Isn't possible to use a temporary table like in the function "ipv4_sysctl_local_port_range" ?
@@ -851,6 +851,20 @@ static int inet6_net_init(struct net *nenet->ipv6.sysctl.ip6_rt_min_advmss=IPV6_MIN_MTU-20-40;net->ipv6.sysctl.icmpv6_time=1*HZ;+mem_reserve_init(&net->ipv6.ip6_rt_reserve,"IPv6 route cache",+&net_rx_reserve);+/*+*XXX:requiresthatnet->ipv6.ip6_dst_opsisalreadyset-up+*butafaiktitsimpossibletoorderthevarious+*pernet_subsyscallssothatthisoneisdoneafter+*ip6_route_net_init().+*/
As this code seems related to the routes, is there a particular reason to not put it at the end of "ip6_route_net_init" function ? You will be sure "net->ipv6.ip6_dst_ops is already set-up", no ?
quoted hunk
+ err = mem_reserve_kmem_cache_set(&net->ipv6.ip6_rt_reserve,+ net->ipv6.ip6_dst_ops.kmem_cachep,+ net->ipv6.sysctl.ip6_rt_max_size);+ if (err)+ goto reserve_fail;+ #ifdef CONFIG_PROC_FS err = udp6_proc_init(net); if (err)
@@ -861,8 +875,8 @@ static int inet6_net_init(struct net *ne err = ac6_proc_init(net); if (err) goto proc_ac6_fail;-out: #endif+out: return err; #ifdef CONFIG_PROC_FS
From: Peter Zijlstra <hidden> Date: 2008-10-01 18:57:23
On Wed, 2008-10-01 at 13:38 +0200, Daniel Lezcano wrote:
I removed a big portion of code because the remarks below apply to the
rest of the code.
quoted
+static int sysctl_intvec_route(struct ctl_table *table,+ int __user *name, int nlen,+ void __user *oldval, size_t __user *oldlenp,+ void __user *newval, size_t newlen)+{+ struct net *net = current->nsproxy->net_ns;
I think you can use the container_of and get rid of using
current->nsproxy->net_ns.
struct net *net = container_of(table->data, struct net,
ipv6.sysctl.ip6_rt_max_size);
D'oh - why didn't I think of that... yes very nice.
quoted
+ int write = (newval && newlen);+ int new_size, ret;++ mutex_lock(&net->ipv6.sysctl.ip6_rt_lock);++ if (write)+ table->data = &new_size;++ ret = sysctl_intvec(table, name, nlen, oldval, oldlenp, newval, newlen);++ if (!ret && write) {+ ret = mem_reserve_kmem_cache_set(&net->ipv6.ip6_rt_reserve,+ net->ipv6.ip6_dst_ops.kmem_cachep, new_size);+ if (!ret)+ net->ipv6.sysctl.ip6_rt_max_size = new_size;+ }++ if (write)+ table->data = &net->ipv6.sysctl.ip6_rt_max_size;++ mutex_unlock(&net->ipv6.sysctl.ip6_rt_lock);++ return ret;+}
Dancing with the table->data looks safe but it is not very nice.
Isn't possible to use a temporary table like in the function
"ipv4_sysctl_local_port_range" ?
@@ -851,6 +851,20 @@ static int inet6_net_init(struct net *nenet->ipv6.sysctl.ip6_rt_min_advmss=IPV6_MIN_MTU-20-40;net->ipv6.sysctl.icmpv6_time=1*HZ;+mem_reserve_init(&net->ipv6.ip6_rt_reserve,"IPv6 route cache",+&net_rx_reserve);+/*+*XXX:requiresthatnet->ipv6.ip6_dst_opsisalreadyset-up+*butafaiktitsimpossibletoorderthevarious+*pernet_subsyscallssothatthisoneisdoneafter+*ip6_route_net_init().+*/
As this code seems related to the routes, is there a particular reason
to not put it at the end of "ip6_route_net_init" function ? You will be
sure "net->ipv6.ip6_dst_ops is already set-up", no ?
Ah, the problem is that I need both dst_ops and ip6_rt_max_size set.
The former is set in ip6_route_net_init() while the later is set in
inet6_net_init(), both are registered pernet_ops without specified
order.
So where exactly do I hook in?
quoted
+ err = mem_reserve_kmem_cache_set(&net->ipv6.ip6_rt_reserve,+ net->ipv6.ip6_dst_ops.kmem_cachep,+ net->ipv6.sysctl.ip6_rt_max_size);+ if (err)+ goto reserve_fail;+ #ifdef CONFIG_PROC_FS err = udp6_proc_init(net); if (err)
@@ -861,8 +875,8 @@ static int inet6_net_init(struct net *ne err = ac6_proc_init(net); if (err) goto proc_ac6_fail;-out: #endif+out: return err; #ifdef CONFIG_PROC_FS