@@ -0,0 +1,16 @@+#+# IFE subsystem configuration+#++menuconfigNET_IFE+depends onNET+tristate"Inter-FE based on IETF ForCES InterFE LFB"+defaultn+help+SayYheretoaddsupportofIFEencapsulationprotocol+Fordetailsrefertonetdev01paper:+"Distributing Linux Traffic Control Classifier-Action Subsystem"+Authors:JamalHadiSalimandDamasceneM.Joachimpillai++Tocompilethissupportasamodule,chooseMhere:themodulewill+becalledife.
@@ -0,0 +1,147 @@+/*+*net/ife/ife.cInter-FEprotocolbasedonForCESWGInterFELFB+*+*Referto:+*draft-ietf-forces-interfelfb-03+*and+*netdev01paper:+*"Distributing Linux Traffic Control Classifier-Action+*Subsystem"+*Authors:JamalHadiSalimandDamasceneM.Joachimpillai+*+*Thisprogramisfreesoftware;youcanredistributeitand/or+*modifyitunderthetermsoftheGNUGeneralPublicLicense+*aspublishedbytheFreeSoftwareFoundation;eitherversion+*2oftheLicense,or(atyouroption)anylaterversion.+*+*/++#include<linux/types.h>+#include<linux/kernel.h>+#include<linux/string.h>+#include<linux/errno.h>+#include<linux/skbuff.h>+#include<linux/rtnetlink.h>+#include<linux/module.h>+#include<linux/init.h>+#include<net/net_namespace.h>+#include<net/netlink.h>+#include<net/pkt_sched.h>+#include<linux/etherdevice.h>+#include<net/ife.h>++void*ife_encode(structsk_buff*skb,u16metalen)+{+/* OUTERHDR:TOTMETALEN:{TLVHDR:Metadatum:TLVHDR..}:ORIGDATA+*whereORIGDATA=originalethernetheader...+*/+inthdrm=metalen+IFE_METAHDRLEN;+inttotal_push=hdrm+skb->dev->hard_header_len;+structethhdr*iethh;/* inner ether header */+intskboff=0;+interr;++err=skb_cow_head(skb,total_push);+if(unlikely(err))+returnNULL;++iethh=(structethhdr*)skb->data;++__skb_push(skb,total_push);+memcpy(skb->data,iethh,skb->dev->hard_header_len);+skb_reset_mac_header(skb);+skboff+=skb->dev->hard_header_len;++/* total metadata length */+metalen+=IFE_METAHDRLEN;+metalen=htons(metalen);+memcpy((skb->data+skboff),&metalen,IFE_METAHDRLEN);+skboff+=IFE_METAHDRLEN;++returnskb->data+skboff;+}+EXPORT_SYMBOL_GPL(ife_encode);++structifeheadr{+__be16metalen;+u8tlv_data[];+};++void*ife_decode(structsk_buff*skb,u16*metalen)+{+structifeheadr*ifehdr;+inttotal_pull;+u16ifehdrln;++ifehdr=(structifeheadr*)(skb->data+skb->dev->hard_header_len);+ifehdrln=ifehdr->metalen;+ifehdrln=ntohs(ifehdrln);+total_pull=skb->dev->hard_header_len+ifehdrln;++if(unlikely(ifehdrln<2))+returnNULL;++if(unlikely(!pskb_may_pull(skb,total_pull)))+returnNULL;++skb_set_mac_header(skb,total_pull);+__skb_pull(skb,total_pull);+*metalen=ifehdrln-IFE_METAHDRLEN;++return&ifehdr->tlv_data;+}+EXPORT_SYMBOL_GPL(ife_decode);++structmeta_tlvhdr{+__be16type;+__be16len;+};++/* Caller takes care of presenting data in network order+*/+void*ife_tlv_meta_decode(void*skbdata,u16*attrtype,u16*dlen,u16*totlen)+{+structmeta_tlvhdr*tlv=(structmeta_tlvhdr*)skbdata;++*dlen=ntohs(tlv->len)-NLA_HDRLEN;+*attrtype=ntohs(tlv->type);++if(totlen)+*totlen=nla_total_size(*dlen);++returnskbdata+sizeof(structmeta_tlvhdr);+}+EXPORT_SYMBOL_GPL(ife_tlv_meta_decode);++void*ife_tlv_meta_next(void*skbdata)+{+structmeta_tlvhdr*tlv=(structmeta_tlvhdr*)skbdata;+u16tlvlen=tlv->len;++tlvlen=ntohs(tlvlen);+tlvlen=NLA_ALIGN(tlvlen);++returnskbdata+tlvlen;+}+EXPORT_SYMBOL_GPL(ife_tlv_meta_next);++/* Caller takes care of presenting data in network order+*/+intife_tlv_meta_encode(void*skbdata,u16attrtype,u16dlen,constvoid*dval)+{+u32*tlv=(u32*)(skbdata);+u16totlen=nla_total_size(dlen);/*alignment + hdr */+char*dptr=(char*)tlv+NLA_HDRLEN;+u32htlv=attrtype<<16|(dlen+NLA_HDRLEN);++*tlv=htonl(htlv);+memset(dptr,0,totlen-NLA_HDRLEN);+memcpy(dptr,dval,dlen);++returntotlen;+}+EXPORT_SYMBOL_GPL(ife_tlv_meta_encode);++MODULE_AUTHOR("Jamal Hadi Salim (2015)");+MODULE_DESCRIPTION("Inter-FE LFB action");+MODULE_LICENSE("GPL");
From: Yotam Gigi <yotam.gi@gmail.com>
- IFE_META_IFINDEX: Allow to pass ifindex value as part of the ife
metadata
- IFE_META_ORIG_SIZE: Allow to pass the original packet size as part of
the ife metadata. Can be used in case that the packet is truncated
- IFE_META_SIZE: Allow to pass the size of the encapsulated packet as
part of the ife metadata
Signed-off-by: Yotam Gigi <redacted>
Signed-off-by: Jiri Pirko <redacted>
---
include/uapi/linux/ife.h | 3 +++
1 file changed, 3 insertions(+)
@@ -3,6 +3,7 @@#include<linux/types.h>#include<linux/pkt_cls.h>+#include<linux/ife.h>#define TCA_ACT_IFE 25/* Flag bits for now just encoding/decoding; mutually exclusive */
@@ -766,27 +741,10 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a,returnTC_ACT_SHOT;}-err=skb_cow_head(skb,hdrm);-if(unlikely(err)){-ife->tcf_qstats.drops++;-spin_unlock(&ife->tcf_lock);-returnTC_ACT_SHOT;-}-if(!(at&AT_EGRESS))skb_push(skb,skb->dev->hard_header_len);-iethh=(structethhdr*)skb->data;-__skb_push(skb,hdrm);-memcpy(skb->data,iethh,skb->mac_len);-skb_reset_mac_header(skb);-oethh=eth_hdr(skb);--/*total metadata length */-metalen+=IFE_METAHDRLEN;-metalen=htons(metalen);-memcpy((skb->data+skboff),&metalen,IFE_METAHDRLEN);-skboff+=IFE_METAHDRLEN;+ife_meta=ife_encode(skb,metalen);/* XXX: we dont have a clever way of telling encode to*notrepeatsomeofthecomputationsthataredoneby
From: Yotam Gigi <yotam.gi@gmail.com>
This action allow the user to sample traffic matched by tc classifier.
The sampling consists of choosing packets randomly, truncating them,
adding some informative metadata regarding the interface and the original
packet size and mark them with specific mark, to allow further tc rules to
match and process. The marked sample packets are then injected into the
device ingress qdisc using netif_receive_skb.
The packets metadata is packed using the ife encapsulation protocol, and
the outer packet's ethernet dest, source and eth_type, along with the
rate, mark and the optional truncation size can be configured from
userspace.
Example:
To sample ingress traffic from interface eth1, and redirect the sampled
the sampled packets to interface dummy0, one may use the commands:
tc qdisc add dev eth1 handle ffff: ingress
tc filter add dev eth1 parent ffff: \
matchall action sample rate 12 mark 17
tc filter add parent ffff: dev eth1 protocol all \
u32 match mark 172 0xff
action mirred egress redirect dev dummy0
Where the first command adds an ingress qdisc and the second starts
sampling every 12'th packet on dev eth0 and marks the sampled packets with
17. The command third catches the sampled packets, which are marked with
17, and redirects them to dev dummy0.
Signed-off-by: Yotam Gigi <redacted>
Signed-off-by: Jiri Pirko <redacted>
---
include/net/tc_act/tc_sample.h | 88 ++++++++++
include/uapi/linux/tc_act/Kbuild | 1 +
include/uapi/linux/tc_act/tc_sample.h | 31 ++++
net/sched/Kconfig | 13 ++
net/sched/Makefile | 1 +
net/sched/act_sample.c | 300 ++++++++++++++++++++++++++++++++++
6 files changed, 434 insertions(+)
create mode 100644 include/net/tc_act/tc_sample.h
create mode 100644 include/uapi/linux/tc_act/tc_sample.h
create mode 100644 net/sched/act_sample.c
@@ -0,0 +1,31 @@+#ifndef __LINUX_TC_SAMPLE_H+#define __LINUX_TC_SAMPLE_H++#include<linux/types.h>+#include<linux/pkt_cls.h>+#include<linux/if_ether.h>++#define TCA_ACT_SAMPLE 26++structtc_sample{+tc_gen;+__u32rate;/* sample rate */+__u32mark;/* mark to put on the sampled packets */+booltruncate;/* whether to truncate the packets */+__u32trunc_size;/* truncation size */+__u8eth_dst[ETH_ALEN];/* encapsulated mac destination */+__u8eth_src[ETH_ALEN];/* encapsulated mac source */+booleth_type_set;/* whether to overrid ethtype */+__u16eth_type;/* encapsulated mac ethtype */+};++enum{+TCA_SAMPLE_UNSPEC,+TCA_SAMPLE_TM,+TCA_SAMPLE_PARMS,+TCA_SAMPLE_PAD,+__TCA_SAMPLE_MAX+};+#define TCA_SAMPLE_MAX (__TCA_SAMPLE_MAX - 1)++#endif
@@ -0,0 +1,300 @@+/*+*net/sched/act_sample.cpacketsamplingtcaction+*+*Thisprogramisfreesoftware;youcanredistributeitand/or+*modifyitunderthetermsoftheGNUGeneralPublicLicense+*aspublishedbytheFreeSoftwareFoundation;eitherversion+*2oftheLicense,or(atyouroption)anylaterversion.+*+*Authors:YotamGigi<yotamg@mellanox.com>(2016)+*+*/++#include<linux/types.h>+#include<linux/kernel.h>+#include<linux/string.h>+#include<linux/errno.h>+#include<linux/skbuff.h>+#include<linux/rtnetlink.h>+#include<linux/module.h>+#include<linux/init.h>+#include<linux/gfp.h>+#include<net/net_namespace.h>+#include<net/netlink.h>+#include<net/pkt_sched.h>+#include<linux/tc_act/tc_sample.h>+#include<net/tc_act/tc_sample.h>+#include<net/ife.h>++#include<linux/if_arp.h>++#define SAMPLE_TAB_MASK 7+staticintsample_net_id;+staticstructtc_action_opsact_sample_ops;++staticconststructnla_policysample_policy[TCA_SAMPLE_MAX+1]={+[TCA_SAMPLE_PARMS]={.len=sizeof(structtc_sample)},+};++staticinttcf_sample_init(structnet*net,structnlattr*nla,+structnlattr*est,structtc_action**a,intovr,+intbind)+{+structtc_action_net*tn=net_generic(net,sample_net_id);+structnlattr*tb[TCA_SAMPLE_MAX+1];+structtc_sample*parm;+structtcf_sample*s;+intret;+boolexists=false;++if(!nla)+return-EINVAL;+ret=nla_parse_nested(tb,TCA_SAMPLE_MAX,nla,sample_policy);+if(ret<0)+returnret;+if(!tb[TCA_SAMPLE_PARMS])+return-EINVAL;+parm=nla_data(tb[TCA_SAMPLE_PARMS]);++exists=tcf_hash_check(tn,parm->index,a,bind);+if(exists&&bind)+return0;++if(!exists){+ret=tcf_hash_create(tn,parm->index,est,a,+&act_sample_ops,bind,false);+if(ret)+returnret;+ret=ACT_P_CREATED;+}else{+tcf_hash_release(*a,bind);+if(!ovr)+return-EEXIST;+}+s=to_sample(*a);++ASSERT_RTNL();+s->tcf_action=parm->action;+s->rate=parm->rate;+s->mark=parm->mark;+s->truncate=parm->truncate;+s->trunc_size=parm->trunc_size;+s->eth_type=parm->eth_type;+s->eth_type_set=parm->eth_type_set;+s->packet_counter=0;++if(parm->eth_dst)+ether_addr_copy(s->eth_dst,parm->eth_dst);+else+eth_zero_addr(s->eth_dst);+if(parm->eth_src)+ether_addr_copy(s->eth_src,parm->eth_src);+else+eth_zero_addr(s->eth_src);++if(ret==ACT_P_CREATED)+tcf_hash_insert(tn,*a);+returnret;+}++staticbooldev_ok_push(structnet_device*dev)+{+switch(dev->type){+caseARPHRD_TUNNEL:+caseARPHRD_TUNNEL6:+caseARPHRD_SIT:+caseARPHRD_IPGRE:+caseARPHRD_VOID:+caseARPHRD_NONE:+returnfalse;+default:+returntrue;+}+}++structethhdr*sample_packet_pack(structsk_buff*skb,+structsample_packet_metadata*metadata)+{+intsample_size;+intorig_size;+void*ifetlv;+intifindex;+u16metalen;++metalen=nla_total_size(sizeof(metadata->ifindex))++nla_total_size(sizeof(metadata->orig_size))++nla_total_size(sizeof(metadata->sample_size));++ifindex=htonl(metadata->ifindex);+orig_size=htonl(metadata->orig_size);+sample_size=htonl(metadata->sample_size);++ifetlv=ife_encode(skb,metalen);+if(!ifetlv)+returnNULL;++ifetlv+=ife_tlv_meta_encode(ifetlv,IFE_META_IFINDEX,+sizeof(ifindex),&ifindex);++ifetlv+=ife_tlv_meta_encode(ifetlv,IFE_META_ORIGSIZE,+sizeof(orig_size),&orig_size);++ifetlv+=ife_tlv_meta_encode(ifetlv,IFE_META_SIZE,+sizeof(sample_size),&sample_size);++return(structethhdr*)skb->data;+}+EXPORT_SYMBOL(sample_packet_pack);++staticinttcf_sample(structsk_buff*skb,conststructtc_action*a,+structtcf_result*res)+{+structtcf_sample*s=to_sample(a);+structsample_packet_metadatametadata;+staticstructethhdr*ethhdr;+structsk_buff*skb2;+intretval;+u32at;++tcf_lastuse_update(&s->tcf_tm);+bstats_cpu_update(this_cpu_ptr(s->common.cpu_bstats),skb);++rcu_read_lock();+retval=READ_ONCE(s->tcf_action);++if(++s->packet_counter%s->rate==0){+skb2=skb_copy(skb,GFP_ATOMIC);+if(!skb2)+gotoout;++if(s->truncate)+skb_trim(skb2,s->trunc_size);++at=G_TC_AT(skb->tc_verd);+skb2->mac_len=skb->mac_len;++/* on ingress, the mac header gets poped, so push it back */+if(!(at&AT_EGRESS)&&dev_ok_push(skb->dev))+skb_push(skb2,skb2->mac_len);++metadata.ifindex=skb->dev->ifindex;+metadata.orig_size=skb->len+skb->dev->hard_header_len;+metadata.sample_size=skb2->len;+ethhdr=sample_packet_pack(skb2,(void*)&metadata);+if(!ethhdr)+gotoout;++if(!is_zero_ether_addr(s->eth_src))+ether_addr_copy(ethhdr->h_source,s->eth_src);+if(!is_zero_ether_addr(s->eth_dst))+ether_addr_copy(ethhdr->h_dest,s->eth_dst);+if(s->eth_type_set)+ethhdr->h_proto=htons(s->eth_type);++skb2->mark=s->mark;+netif_receive_skb(skb2);++/* mirror is always swallowed */+skb2->tc_verd=SET_TC_FROM(skb2->tc_verd,at);+}+out:+rcu_read_unlock();++returnretval;+}++staticinttcf_sample_dump(structsk_buff*skb,structtc_action*a,+intbind,intref)+{+unsignedchar*b=skb_tail_pointer(skb);+structtcf_sample*s=to_sample(a);+structtc_sampleopt={+.index=s->tcf_index,+.action=s->tcf_action,+.refcnt=s->tcf_refcnt-ref,+.bindcnt=s->tcf_bindcnt-bind,+.rate=s->rate,+.mark=s->mark,+.trunc_size=s->trunc_size,+.truncate=s->truncate,+};+structtcf_tt;++if(nla_put(skb,TCA_SAMPLE_PARMS,sizeof(opt),&opt))+gotonla_put_failure;++tcf_tm_dump(&t,&s->tcf_tm);+if(nla_put_64bit(skb,TCA_SAMPLE_TM,sizeof(t),&t,TCA_SAMPLE_PAD))+gotonla_put_failure;+returnskb->len;++nla_put_failure:+nlmsg_trim(skb,b);+return-1;+}++staticinttcf_sample_walker(structnet*net,structsk_buff*skb,+structnetlink_callback*cb,inttype,+conststructtc_action_ops*ops)+{+structtc_action_net*tn=net_generic(net,sample_net_id);++returntcf_generic_walker(tn,skb,cb,type,ops);+}++staticinttcf_sample_search(structnet*net,structtc_action**a,u32index)+{+structtc_action_net*tn=net_generic(net,sample_net_id);++returntcf_hash_search(tn,a,index);+}++staticstructtc_action_opsact_sample_ops={+.kind="sample",+.type=TCA_ACT_SAMPLE,+.owner=THIS_MODULE,+.act=tcf_sample,+.dump=tcf_sample_dump,+.init=tcf_sample_init,+.walk=tcf_sample_walker,+.lookup=tcf_sample_search,+.size=sizeof(structtcf_sample),+};++static__net_initintsample_init_net(structnet*net)+{+structtc_action_net*tn=net_generic(net,sample_net_id);++returntc_action_net_init(tn,&act_sample_ops,SAMPLE_TAB_MASK);+}++staticvoid__net_exitsample_exit_net(structnet*net)+{+structtc_action_net*tn=net_generic(net,sample_net_id);++tc_action_net_exit(tn);+}++staticstructpernet_operationssample_net_ops={+.init=sample_init_net,+.exit=sample_exit_net,+.id=&sample_net_id,+.size=sizeof(structtc_action_net),+};++MODULE_AUTHOR("Yotam Gigi (2016)");+MODULE_DESCRIPTION("Packet sampling action");+MODULE_LICENSE("GPL");++staticint__initsample_init_module(void)+{+returntcf_register_action(&act_sample_ops,&sample_net_ops);+}++staticvoid__exitsample_cleanup_module(void)+{+tcf_unregister_action(&act_sample_ops,&sample_net_ops);+}++module_init(sample_init_module);+module_exit(sample_cleanup_module);
From: Yotam Gigi <yotam.gi@gmail.com>
The MPSC register allows to configure ingress packet sampling on specific
port of the mlxsw device. The sampled packets are then trapped via
PKT_SAMPLE trap.
Signed-off-by: Yotam Gigi <redacted>
Signed-off-by: Jiri Pirko <redacted>
---
drivers/net/ethernet/mellanox/mlxsw/reg.h | 43 +++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+)
From: Yotam Gigi <yotam.gi@gmail.com>
Using the MPSC regiter, add the functions that configure port packets
sampling in hardware and the necessary datatypes in the mlxsw_sp_port
struct. In addition, add the necessary trap for sampled packets and
integrate with matchall offloading to allow offloading of the sample tc
action.
The current offload support is for the tc command:
tc filter add dev <DEV> parent ffff: \
matchall \
action sample rate <RATE> mark <MARK> [trunc <SIZE>] \
[src <SADDR>] [dst <DADDR>] [type <TYPE>]
Where only ingress qdiscs are supported, and only a combination of
matchall classifier and sample action will lead to activating hardware
packet sampling.
Signed-off-by: Yotam Gigi <redacted>
Signed-off-by: Jiri Pirko <redacted>
---
drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 116 +++++++++++++++++++++++--
drivers/net/ethernet/mellanox/mlxsw/spectrum.h | 11 +++
drivers/net/ethernet/mellanox/mlxsw/trap.h | 1 +
3 files changed, 122 insertions(+), 6 deletions(-)
@@ -2749,6 +2850,9 @@ static void mlxsw_sp_rx_listener_mark_func(struct sk_buff *skb, u8 local_port,staticconststructmlxsw_rx_listenermlxsw_sp_rx_listener[]={MLXSW_SP_RXL(mlxsw_sp_rx_listener_func,FDB_MC,TRAP_TO_CPU),+MLXSW_SP_RXL(mlxsw_sp_rx_listener_sample_func,PKT_SAMPLE,+MIRROR_TO_CPU),+/* Traps for specific L2 packet types, not trapped as FDB MC */MLXSW_SP_RXL(mlxsw_sp_rx_listener_func,STP,TRAP_TO_CPU),MLXSW_SP_RXL(mlxsw_sp_rx_listener_func,LACP,TRAP_TO_CPU),
Thu, Oct 13, 2016 at 09:29:57AM CEST, roopa@cumulusnetworks.com wrote:
On 10/12/16, 5:41 AM, Jiri Pirko wrote:
quoted
From: Jiri Pirko <redacted>
Add the sample tc action, which allows to sample packet matching
a classifier. The sample action peeks randomly packets, duplicates them,
truncates them and adds informative metadata on the packet, for example,
the input interface and the original packet length. The sampled packets
are marked to allow matching them and redirecting them to a specific
collector device.
The sampled packets metadata is packed using ife encapsulation. To do
that, this patch-set extracts ife logics from the tc_ife action into an
independent ife module, and uses that functionality to pack the metadata.
To include all the needed metadata, this patch-set introduces some new
IFE_META tlv types.
In addition, Add the support for offloading the matchall-sample tc command
in the Mellanox mlxsw driver, for ingress qdiscs.
Yotam Gigi (6):
Introduce ife encapsulation module
act_ife: Change to use ife module
ife: Introduce new metadata tlv types
Introduce sample tc action
mlxsw: reg: add the Monitoring Packet Sampling Configuration Register
mlxsw: packet sample: Add packet sample offloading support
we spoke with yotam about this at netdev1.2. and also remember speaking about this on our switchdev calls:
Today our driver uses NFLOG to log packets to a netlink socket and hsflowd supported by the sflow
people (at http://www.sflow.net/) is capable of reading from a nflog socket. NFLOG has the required netlink
attribute markers for packet header/data (which we can possibly extend). We could also add nflog like action
in tc if needed.
sflow agents like hsflowd are capable of sending packets to an external collector with the required sflow header.
Instead of re-inventing a new API for sflow, would be better to standardize/unify on existing mechanisms.
Also, this patch series requires a new device to be created which can be avoided if we used
existing mechanisms like NFLOG.
When I was first thinking about re-using NFLOG, it seemed like an
abusal. We need to call it from driver directly, which sounds odd.
However, since we use sample_packet_pack function to wrap it up, the
NFLOG is called from the tc action code, it does not look bad.
Yet still, this has nothing in common with netfilter, only using it's
log facilities. That is odd.
I think that the IFE ways is way more clear and generic and not-abusing.
However you are right the NFLOG way has advantage of existing user
component. I'm not sure how to do this :(
From: Jiri Pirko <redacted>
Add the sample tc action, which allows to sample packet matching
a classifier. The sample action peeks randomly packets, duplicates them,
truncates them and adds informative metadata on the packet, for example,
the input interface and the original packet length. The sampled packets
are marked to allow matching them and redirecting them to a specific
collector device.
The sampled packets metadata is packed using ife encapsulation. To do
that, this patch-set extracts ife logics from the tc_ife action into an
independent ife module, and uses that functionality to pack the metadata.
To include all the needed metadata, this patch-set introduces some new
IFE_META tlv types.
In addition, Add the support for offloading the matchall-sample tc command
in the Mellanox mlxsw driver, for ingress qdiscs.
Yotam Gigi (6):
Introduce ife encapsulation module
act_ife: Change to use ife module
ife: Introduce new metadata tlv types
Introduce sample tc action
mlxsw: reg: add the Monitoring Packet Sampling Configuration Register
mlxsw: packet sample: Add packet sample offloading support
we spoke with yotam about this at netdev1.2. and also remember speaking about this on our switchdev calls:
Today our driver uses NFLOG to log packets to a netlink socket and hsflowd supported by the sflow
people (at http://www.sflow.net/) is capable of reading from a nflog socket. NFLOG has the required netlink
attribute markers for packet header/data (which we can possibly extend). We could also add nflog like action
in tc if needed.
sflow agents like hsflowd are capable of sending packets to an external collector with the required sflow header.
Instead of re-inventing a new API for sflow, would be better to standardize/unify on existing mechanisms.
Also, this patch series requires a new device to be created which can be avoided if we used
existing mechanisms like NFLOG.
Thu, Oct 13, 2016 at 01:49:07PM CEST, jhs@mojatatu.com wrote:
On 16-10-13 04:48 AM, Jiri Pirko wrote:
quoted
Thu, Oct 13, 2016 at 09:29:57AM CEST, roopa@cumulusnetworks.com wrote:
quoted
On 10/12/16, 5:41 AM, Jiri Pirko wrote:
quoted
From: Jiri Pirko <redacted>
[..]
quoted
quoted
we spoke with yotam about this at netdev1.2. and also remember speaking about this on our switchdev calls:
Today our driver uses NFLOG to log packets to a netlink socket and hsflowd supported by the sflow
people (at http://www.sflow.net/) is capable of reading from a nflog socket. NFLOG has the required netlink
attribute markers for packet header/data (which we can possibly extend). We could also add nflog like action
in tc if needed.
sflow agents like hsflowd are capable of sending packets to an external collector with the required sflow header.
Instead of re-inventing a new API for sflow, would be better to standardize/unify on existing mechanisms.
Also, this patch series requires a new device to be created which can be avoided if we used
existing mechanisms like NFLOG.
When I was first thinking about re-using NFLOG, it seemed like an
abusal. We need to call it from driver directly, which sounds odd.
However, since we use sample_packet_pack function to wrap it up, the
NFLOG is called from the tc action code, it does not look bad.
Yet still, this has nothing in common with netfilter, only using it's
log facilities. That is odd.
Sorry, had not seen the code until now; helps me get perspective.
If you are going to require netfilter just so you can do this - it
sounds so wrong (since you already provides a hook for tc offloading
into the switch for other functions).
+1
Roopa, did you mean eth1 as the new device or did you mean just in
general config requiring a device to be specified or did you mean a new
cpu netdev being needed? I couldnt tell from the patch.
You just have to have some netdev to use to funnel the IFE headered
sample skbs to userspace. A dummy or a tap.
quoted
I think that the IFE ways is way more clear and generic and not-abusing.
However you are right the NFLOG way has advantage of existing user
component. I'm not sure how to do this :(
Can you do NFLOG to user space without requiring netfilter compiled in?
One advantage with IFE is it is a wire protocol - so you can have the
sflow collector/aggregator sit on a different machine (for small cpu
switches makes sense). So modifying the sflow daemon to accept IFE
formatted data is an interesting!
Agreed. For me, that looks like the correct way to do this.
Thu, Oct 13, 2016 at 02:30:19PM CEST, jhs@mojatatu.com wrote:
On 16-10-13 08:10 AM, Jiri Pirko wrote:
quoted
Thu, Oct 13, 2016 at 01:49:07PM CEST, jhs@mojatatu.com wrote:
quoted
On 16-10-13 04:48 AM, Jiri Pirko wrote:
[..]
quoted
quoted
Roopa, did you mean eth1 as the new device or did you mean just in
general config requiring a device to be specified or did you mean a new
cpu netdev being needed? I couldnt tell from the patch.
You just have to have some netdev to use to funnel the IFE headered
sample skbs to userspace. A dummy or a tap.
I see.
So with nflog you get basically a backend using a netlink socket
but in your case you will redirect to tuntap for the case of local
sflow but some other device for remote? I am assuming using dummy
would require a packet socket as means of retrieving the data.
Correct. The idea is that the userspace app would create a tap device,
setup the sampling packets to be sent there and recieve them
over chardev. Or the remote delivery could be use to push the sampling
packet to a remote host.
If you take the structuring of the metadata that nflog uses it should
be easy to transpose.
Yes, we do it with IFE, this patchset implements that.
To Roopa's point, however: Would it not make sense to support nflog
(in addition?).
cheers,
jamal
From: Jamal Hadi Salim <jhs@mojatatu.com> Date: 2016-10-13 12:45:49
On 16-10-13 08:10 AM, Jiri Pirko wrote:
Thu, Oct 13, 2016 at 01:49:07PM CEST, jhs@mojatatu.com wrote:
quoted
On 16-10-13 04:48 AM, Jiri Pirko wrote:
[..]
quoted
Roopa, did you mean eth1 as the new device or did you mean just in
general config requiring a device to be specified or did you mean a new
cpu netdev being needed? I couldnt tell from the patch.
You just have to have some netdev to use to funnel the IFE headered
sample skbs to userspace. A dummy or a tap.
I see.
So with nflog you get basically a backend using a netlink socket
but in your case you will redirect to tuntap for the case of local
sflow but some other device for remote? I am assuming using dummy
would require a packet socket as means of retrieving the data.
If you take the structuring of the metadata that nflog uses it should
be easy to transpose.
To Roopa's point, however: Would it not make sense to support nflog
(in addition?).
cheers,
jamal
From: Jamal Hadi Salim <jhs@mojatatu.com> Date: 2016-10-13 12:58:50
On 16-10-13 04:48 AM, Jiri Pirko wrote:
Thu, Oct 13, 2016 at 09:29:57AM CEST, roopa@cumulusnetworks.com wrote:
quoted
On 10/12/16, 5:41 AM, Jiri Pirko wrote:
quoted
From: Jiri Pirko <redacted>
[..]
quoted
we spoke with yotam about this at netdev1.2. and also remember speaking about this on our switchdev calls:
Today our driver uses NFLOG to log packets to a netlink socket and hsflowd supported by the sflow
people (at http://www.sflow.net/) is capable of reading from a nflog socket. NFLOG has the required netlink
attribute markers for packet header/data (which we can possibly extend). We could also add nflog like action
in tc if needed.
sflow agents like hsflowd are capable of sending packets to an external collector with the required sflow header.
Instead of re-inventing a new API for sflow, would be better to standardize/unify on existing mechanisms.
Also, this patch series requires a new device to be created which can be avoided if we used
existing mechanisms like NFLOG.
When I was first thinking about re-using NFLOG, it seemed like an
abusal. We need to call it from driver directly, which sounds odd.
However, since we use sample_packet_pack function to wrap it up, the
NFLOG is called from the tc action code, it does not look bad.
Yet still, this has nothing in common with netfilter, only using it's
log facilities. That is odd.
Sorry, had not seen the code until now; helps me get perspective.
If you are going to require netfilter just so you can do this - it
sounds so wrong (since you already provides a hook for tc offloading
into the switch for other functions).
Roopa, did you mean eth1 as the new device or did you mean just in
general config requiring a device to be specified or did you mean a new
cpu netdev being needed? I couldnt tell from the patch.
I think that the IFE ways is way more clear and generic and not-abusing.
However you are right the NFLOG way has advantage of existing user
component. I'm not sure how to do this :(
Can you do NFLOG to user space without requiring netfilter compiled in?
One advantage with IFE is it is a wire protocol - so you can have the
sflow collector/aggregator sit on a different machine (for small cpu
switches makes sense). So modifying the sflow daemon to accept IFE
formatted data is an interesting!
cheers,
jamal
On Thu, Oct 13, 2016 at 5:45 AM, Jiri Pirko [off-list ref] wrote:
Thu, Oct 13, 2016 at 02:30:19PM CEST, jhs@mojatatu.com wrote:
quoted
On 16-10-13 08:10 AM, Jiri Pirko wrote:
quoted
Thu, Oct 13, 2016 at 01:49:07PM CEST, jhs@mojatatu.com wrote:
quoted
On 16-10-13 04:48 AM, Jiri Pirko wrote:
[..]
quoted
quoted
Roopa, did you mean eth1 as the new device or did you mean just in
general config requiring a device to be specified or did you mean a new
cpu netdev being needed? I couldnt tell from the patch.
You just have to have some netdev to use to funnel the IFE headered
sample skbs to userspace. A dummy or a tap.
I see.
So with nflog you get basically a backend using a netlink socket
but in your case you will redirect to tuntap for the case of local
sflow but some other device for remote? I am assuming using dummy
would require a packet socket as means of retrieving the data.
Correct. The idea is that the userspace app would create a tap device,
setup the sampling packets to be sent there and recieve them
over chardev. Or the remote delivery could be use to push the sampling
packet to a remote host.
quoted
If you take the structuring of the metadata that nflog uses it should
be easy to transpose.
Yes, we do it with IFE, this patchset implements that.
quoted
To Roopa's point, however: Would it not make sense to support nflog
(in addition?).
[sorry responding to all conversations so far here]
using ife for delivery of sampled packets to remote is a good option
to have if you have users.
so far I have seen agents collecting samples locally and have their
own protocol to ship them
to a collector (example sflow). Just bringing that up so that we don't
optimize for the less common case
and make the common case difficult to use :).
In my conversations with the sflow people (founders) and others,
netlink as a mechanism for sampled packet
delivery (similar to ulog/nflog) has proven useful and they see it as
a great API to standardize on going forward (Given they are
already using netlink for collecting other samples like stats etc).
something to thing about.
The people I know collecting samples are happy with having netfilter.
agreed that tc already has an existing hw
offload mechanism. and I was not suggesting giving up on tc either.
and also to jiri, agree, I don't think logging from the driver is a
good option. I was merely suggesting
having a similar option without the need for a new collector device.
The three steps in the patch series to collect samples + a device
seems a bit heavy weight.
but, if you think you have users for it, sure. having multiple api's
is also an option.
But api's come with a cost of maintaining them for ever.
From: Yotam Gigi <yotam.gi@gmail.com>
This action allow the user to sample traffic matched by tc classifier.
The sampling consists of choosing packets randomly, truncating them,
adding some informative metadata regarding the interface and the original
packet size and mark them with specific mark, to allow further tc rules to
match and process. The marked sample packets are then injected into the
device ingress qdisc using netif_receive_skb.
The packets metadata is packed using the ife encapsulation protocol, and
the outer packet's ethernet dest, source and eth_type, along with the
rate, mark and the optional truncation size can be configured from
userspace.
Example:
To sample ingress traffic from interface eth1, and redirect the sampled
the sampled packets to interface dummy0, one may use the commands:
tc qdisc add dev eth1 handle ffff: ingress
tc filter add dev eth1 parent ffff: \
matchall action sample rate 12 mark 17
tc filter add parent ffff: dev eth1 protocol all \
u32 match mark 172 0xff
action mirred egress redirect dev dummy0
Where the first command adds an ingress qdisc and the second starts
sampling every 12'th packet on dev eth0 and marks the sampled packets with
17. The command third catches the sampled packets, which are marked with
17, and redirects them to dev dummy0.
Signed-off-by: Yotam Gigi <redacted>
Signed-off-by: Jiri Pirko <redacted>
channeling some feedback from Peter Phaal @sflow inline below:
This metadata does not look extensible.. can it be made to ?
With sflow in context, you need a pair of ifindex numbers to encode ingress and egress ports. Ideally you would also include a sequence number and a count of the total number of packets that were candidates for sampling. The OVS implementation is a good example, the metadata includes all the actions applied to the packet in the kernel data path.
[snip]
@@ -0,0 +1,31 @@+#ifndef __LINUX_TC_SAMPLE_H+#define __LINUX_TC_SAMPLE_H++#include<linux/types.h>+#include<linux/pkt_cls.h>+#include<linux/if_ether.h>++#define TCA_ACT_SAMPLE 26++structtc_sample{+tc_gen;+__u32rate;/* sample rate */+__u32mark;/* mark to put on the sampled packets */+booltruncate;/* whether to truncate the packets */+__u32trunc_size;/* truncation size */+__u8eth_dst[ETH_ALEN];/* encapsulated mac destination */+__u8eth_src[ETH_ALEN];/* encapsulated mac source */+booleth_type_set;/* whether to overrid ethtype */+__u16eth_type;/* encapsulated mac ethtype */+};+
this does not look extensible and is part of UAPI ..
Doing the minimum in the kernel and leaving the rest to the user space agent is much more flexible. The user space agent can attach additional metadata and offer more flexibility in forwarding (sFlow uses XDR encoding over UDP and is routable over IPv4/IPv6).
The sampling function isn’t random
if (++s->packet_counter % s->rate == 0) {
This is unsuitable for sFlow, which is specific about the random sampling function required. BPF, OVS, and the
ULOG statistics module include efficient kernel based random sampling functions that could be used instead.
Thanks,
Roopa
From: Yotam Gigi <yotam.gi@gmail.com>
This action allow the user to sample traffic matched by tc classifier.
The sampling consists of choosing packets randomly, truncating them,
adding some informative metadata regarding the interface and the original
packet size and mark them with specific mark, to allow further tc rules to
match and process. The marked sample packets are then injected into the
device ingress qdisc using netif_receive_skb.
The packets metadata is packed using the ife encapsulation protocol, and
the outer packet's ethernet dest, source and eth_type, along with the
rate, mark and the optional truncation size can be configured from
userspace.
Example:
To sample ingress traffic from interface eth1, and redirect the sampled
the sampled packets to interface dummy0, one may use the commands:
tc qdisc add dev eth1 handle ffff: ingress
tc filter add dev eth1 parent ffff: \
matchall action sample rate 12 mark 17
tc filter add parent ffff: dev eth1 protocol all \
u32 match mark 172 0xff
action mirred egress redirect dev dummy0
Where the first command adds an ingress qdisc and the second starts
sampling every 12'th packet on dev eth0 and marks the sampled packets with
17. The command third catches the sampled packets, which are marked with
17, and redirects them to dev dummy0.
Signed-off-by: Yotam Gigi <redacted>
Signed-off-by: Jiri Pirko <redacted>
channeling some feedback from Peter Phaal @sflow inline below:
If it helps, one more thing that came up was using bpf.
They also use bpf filters for pkt sampling in the non-offloaded case:
http://blog.sflow.com/2016/05/berkeley-packet-filter-bpf.html
so, existing apps (like sflow) that care about packet sampling do prefer to use
a socket api for sample delivery: netlink nflog or bpf like socket filters
also, to keep the software and hardware models the same, wondering if ebpf attach
can be a viable option (have not thought about the offloaded case completely yet).
This would give apps more control on attaching sample headers (like sflow) if needed.
thanks,
Roopa
From: Or Gerlitz <hidden> Date: 2016-10-16 10:27:16
On Wed, Oct 12, 2016 at 3:41 PM, Jiri Pirko [off-list ref] wrote:
From: Yotam Gigi <yotam.gi@gmail.com>
This action allow the user to sample traffic matched by tc classifier.
The sampling consists of choosing packets randomly, truncating them,
adding some informative metadata regarding the interface and the original
packet size and mark them with specific mark, to allow further tc rules to
match and process. The marked sample packets are then injected into the
device ingress qdisc using netif_receive_skb.
The packets metadata is packed using the ife encapsulation protocol, and
the outer packet's ethernet dest, source and eth_type, along with the
rate, mark and the optional truncation size can be configured from
userspace.
Example:
To sample ingress traffic from interface eth1, and redirect the sampled
the sampled packets to interface dummy0, one may use the commands:
tc qdisc add dev eth1 handle ffff: ingress
tc filter add dev eth1 parent ffff: \
matchall action sample rate 12 mark 17
tc filter add parent ffff: dev eth1 protocol all \
u32 match mark 172 0xff
action mirred egress redirect dev dummy0
Where the first command adds an ingress qdisc and the second starts
sampling every 12'th packet on dev eth0 and marks the sampled packets with
17. The command third catches the sampled packets, which are marked with
17, and redirects them to dev dummy0.
eth0 --> eth1
command third --> third command
don't we need a re-classify directive for the u32 filter to apply
after the marking done by the matchall rule + sample action
or is that implicit?
@@ -0,0 +1,31 @@+#ifndef __LINUX_TC_SAMPLE_H+#define __LINUX_TC_SAMPLE_H++#include<linux/types.h>+#include<linux/pkt_cls.h>+#include<linux/if_ether.h>++#define TCA_ACT_SAMPLE 26++structtc_sample{+tc_gen;+__u32rate;/* sample rate */+__u32mark;/* mark to put on the sampled packets */+booltruncate;/* whether to truncate the packets */+__u32trunc_size;/* truncation size */+__u8eth_dst[ETH_ALEN];/* encapsulated mac destination */+__u8eth_src[ETH_ALEN];/* encapsulated mac source */+booleth_type_set;/* whether to overrid ethtype */+__u16eth_type;/* encapsulated mac ethtype */+};
overrid --> override
what do you mean by override here, to encapsulate?
consider using 0 as special value, e.g no truncation and no encapsulation
best if you just define the netlink attributes (document on the RHS
the type, see the uapi
for the new tunnel key action) and let the tc action in-kernel code to
decode them directly
into the non UAPI structure. This way you are extendable and also
avoid having two
structs which is sort of confusing.
+static bool dev_ok_push(struct net_device *dev)
+{
+ switch (dev->type) {
+ case ARPHRD_TUNNEL:
+ case ARPHRD_TUNNEL6:
+ case ARPHRD_SIT:
+ case ARPHRD_IPGRE:
+ case ARPHRD_VOID:
+ case ARPHRD_NONE:
+ return false;
+ default:
+ return true;
+ }
+}
+
+static int tcf_sample(struct sk_buff *skb, const struct tc_action *a,
+ struct tcf_result *res)
+{
+ struct tcf_sample *s = to_sample(a);
+ struct sample_packet_metadata metadata;
+ static struct ethhdr *ethhdr;
+ struct sk_buff *skb2;
+ int retval;
+ u32 at;
+
+ tcf_lastuse_update(&s->tcf_tm);
+ bstats_cpu_update(this_cpu_ptr(s->common.cpu_bstats), skb);
+
+ rcu_read_lock();
+ retval = READ_ONCE(s->tcf_action);
+
+ if (++s->packet_counter % s->rate == 0) {
+ skb2 = skb_copy(skb, GFP_ATOMIC);
+ if (!skb2)
+ goto out;
+
+ if (s->truncate)
+ skb_trim(skb2, s->trunc_size);
+
+ at = G_TC_AT(skb->tc_verd);
+ skb2->mac_len = skb->mac_len;
+
+ /* on ingress, the mac header gets poped, so push it back */
+ if (!(at & AT_EGRESS) && dev_ok_push(skb->dev))
+ skb_push(skb2, skb2->mac_len);
+
what's the exact role of the !(at & AT_EGRESS) check?
and if !dev_ok_push(.) - are we just fine to continue here without
that push? maybe
worth documenting that corner a bit
From: Jamal Hadi Salim <jhs@mojatatu.com> Date: 2016-10-17 10:10:27
Some comments:
IIUC, the main struggle seems to be whether the redirect to dummy0
is useful or not? i.e instead of just letting the packets go up the
stack on eth1?
It seems like sflowd needs to read off eth1 via packet socket?
To be backward compatible - supporting that approach seems sensible.
Note:
There is a clear efficiency benefit of both using IFE encoding and
redirecting to dummy0.
1) Redirecting to dummy0 implies you dont need to exercise a bpf
filter around every packet that comes off eth1.
I understand there are probably not millions of pps for this case;
but in a non-offloaded cases it could be millions pps.
And in case of sampling over many ethx devices, you can redirect
samples from many other ethx devices.
So making dummy0 the sflow device is a win.
2) Encaping an IFE header implies a much more efficient bpf filter
(IFE ethertype is an excellent discriminator for bpf).
Additional benefit is as mentioned before - redirecting to a device
means you can send it remotely over ethernet to a more powerful
machine without having to cross kernel-userspace. Redirecting instead
of mirroring to tuntap is also an interesting option.
More comments below (on the sflow person's comment - dont seem him
on the Cc):
On 16-10-15 12:34 PM, Roopa Prabhu wrote:
On 10/12/16, 5:41 AM, Jiri Pirko wrote:
quoted
From: Yotam Gigi <yotam.gi@gmail.com>
quoted
+
+struct sample_packet_metadata {
+ int sample_size;
+ int orig_size;
+ int ifindex;
+};
+
This metadata does not look extensible.. can it be made to ?
Sure it can...
With sflow in context, you need a pair of ifindex numbers to encode ingress and egress ports.
What is the use case for both?
Ideally you would also include a sequence number and a count of the total number of packets
> that were candidates for sampling.
Sequence number may make sense (they will help show a gap if something
gets dropped). But i am not sure about the stats consuming such space.
Stats are something that can be queried (tc stats should have a record
of how many bytes/packets )
The OVS implementation is a good example, the metadata includes all the actions applied
to the packet in the kernel data path.
Again not sure what the use case would be (and why waste such space
especially when you are sending over the wire with such details).
The sampling function isn’t random
if (++s->packet_counter % s->rate == 0) {
This is unsuitable for sFlow, which is specific about the random sampling function required.
BPF, OVS, and the
ULOG statistics module include efficient kernel based random sampling functions that could be used instead.
If i understood correctly, the above is a fallback sampling algorithm.
In the case of the spectrum it already does the sampling in the ASIC
so there is no need to repeat it in software.
Agreed that in that case the sampling approach is not sufficiently
random.
cheers,
jamal
Some comments:
IIUC, the main struggle seems to be whether the redirect to dummy0
is useful or not? i.e instead of just letting the packets go up the
stack on eth1?
yep, correct...given existing workflow for the non-offloaded case is
to receive sample packets via bpf filter on socket or
use netlink as a sample delivery mechanism (NFLOG eg)
It seems like sflowd needs to read off eth1 via packet socket?
To be backward compatible - supporting that approach seems sensible.
Note:
There is a clear efficiency benefit of both using IFE encoding and
redirecting to dummy0.
1) Redirecting to dummy0 implies you dont need to exercise a bpf
filter around every packet that comes off eth1.
I understand there are probably not millions of pps for this case;
but in a non-offloaded cases it could be millions pps.
And in case of sampling over many ethx devices, you can redirect
samples from many other ethx devices.
So making dummy0 the sflow device is a win.
2) Encaping an IFE header implies a much more efficient bpf filter
(IFE ethertype is an excellent discriminator for bpf).
Additional benefit is as mentioned before - redirecting to a device
means you can send it remotely over ethernet to a more powerful
machine without having to cross kernel-userspace. Redirecting instead
of mirroring to tuntap is also an interesting option.
sure, this seems like a good option to have.
generally you have one instance of the sampling agent on a hyper visor or switch.
But, if you have use-cases where monitoring agents run external, sure.
would have preferred if it was optional or an addon and not the default.
Regarding the device, yeah, agree there are pros and cons.
An additional device just to sample packets seems like an overkill.
But, if there is no other other option, and there are benefits to it, no objections.
Hopefully we can add another option on the existing api to skip the device in the future.
On 16-10-15 12:34 PM, Roopa Prabhu wrote:
quoted
On 10/12/16, 5:41 AM, Jiri Pirko wrote:
quoted
From: Yotam Gigi <yotam.gi@gmail.com>
quoted
quoted
+
+struct sample_packet_metadata {
+ int sample_size;
+ int orig_size;
+ int ifindex;
+};
+
This metadata does not look extensible.. can it be made to ?
Sure it can...
quoted
With sflow in context, you need a pair of ifindex numbers to encode ingress and egress ports.
What is the use case for both?
I have heard that most monitoring tools have moved to ingress only sampling because of operational
complexity (use case is sflow). I think hardware also supports ingress and egress only sampling.
better to have an option to reflect that in the api.
quoted
Ideally you would also include a sequence number and a count of the total number of packets
that were candidates for sampling.
Sequence number may make sense (they will help show a gap if something
gets dropped). But i am not sure about the stats consuming such space.
Stats are something that can be queried (tc stats should have a record
of how many bytes/packets )
sure, thats fine.
quoted
The OVS implementation is a good example, the metadata includes all the actions applied
to the packet in the kernel data path.
Again not sure what the use case would be (and why waste such space
especially when you are sending over the wire with such details).
The sampling function isn’t random
if (++s->packet_counter % s->rate == 0) {
This is unsuitable for sFlow, which is specific about the random sampling function required.
BPF, OVS, and the
ULOG statistics module include efficient kernel based random sampling functions that could be used instead.
If i understood correctly, the above is a fallback sampling algorithm.
In the case of the spectrum it already does the sampling in the ASIC
so there is no need to repeat it in software.
Agreed that in that case the sampling approach is not sufficiently
random.
yes. and since the same sampling api will be used for offloaded and non-offloaded case,
the sampling algo here for the non-offloaded case...can do better .. atleast match the existing
api efficiency. We would want people to use the same api for the offload and non-offloaded case.
thanks,
Roopa
+
+struct sample_packet_metadata {
+ int sample_size;
+ int orig_size;
+ int ifindex;
+};
+
This metadata does not look extensible.. can it be made to ?
Sure it can...
more sflow context here... [1]
An extensible metadata scheme is highly desirable when passing data from the dataplane to
the sampling agent in userspace. Looking forward, advanced instrumentation is being
added to data planes and keeping the api future proof will help.
quoted
quoted
With sflow in context, you need a pair of ifindex numbers to encode ingress and egress ports.
What is the use case for both?
I have heard that most monitoring tools have moved to ingress only sampling because of operational
complexity (use case is sflow). I think hardware also supports ingress and egress only sampling.
better to have an option to reflect that in the api.
The reason for having two ifindex numbers is to record the ingress and egress ports (i.e. the path that the packet takes through the datapath/ASIC). You may actually have three ifindex numbers associated with a sample:
1. The data source that made the measurement (on a linux system each bridge has its own ifindex)
2. The ifindex associated with the ingress switch port
3. The ifindex associated with the egress switch port.
All three apply irrespective of sampling direction.
thanks,
Roopa
[1] Additional extended flow attributes have been defined to further extend sFlow packet samples:
http://sflow.org/sflow_tunnels.txt <http://sflow.org/sflow_tunnels.txt>
http://sflow.org/sflow_openflow.txt <http://sflow.org/sflow_openflow.txt>
-----Original Message-----
From: Roopa Prabhu [mailto:roopa@cumulusnetworks.com]
Sent: Tuesday, October 18, 2016 3:17 AM
To: Jamal Hadi Salim <jhs@mojatatu.com>
Cc: Jiri Pirko <jiri@resnulli.us>; netdev@vger.kernel.org; davem@davemloft.net;
Yotam Gigi [off-list ref]; Ido Schimmel [off-list ref]; Elad
Raz [off-list ref]; Nogah Frankel [off-list ref]; Or Gerlitz
[off-list ref]; geert+renesas@glider.be;
stephen@networkplumber.org; xiyou.wangcong@gmail.com; linux@roeck-us.net;
Shrijeet Mukherjee [off-list ref]
Subject: Re: [patch net-next RFC 4/6] Introduce sample tc action
On 10/17/16, 3:10 AM, Jamal Hadi Salim wrote:
quoted
Some comments:
IIUC, the main struggle seems to be whether the redirect to dummy0
is useful or not? i.e instead of just letting the packets go up the
stack on eth1?
yep, correct...given existing workflow for the non-offloaded case is
to receive sample packets via bpf filter on socket or
use netlink as a sample delivery mechanism (NFLOG eg)
quoted
It seems like sflowd needs to read off eth1 via packet socket?
To be backward compatible - supporting that approach seems sensible.
I am not sure whether using socket is backward compatible. hsflowd
expects nflog packets, and ife packets will not be understood.
We planned on adding support on hsflowd in IFE encapsulated packets from
tuntap device.
Did I understand you correctly?
quoted
Note:
There is a clear efficiency benefit of both using IFE encoding and
redirecting to dummy0.
1) Redirecting to dummy0 implies you dont need to exercise a bpf
filter around every packet that comes off eth1.
I understand there are probably not millions of pps for this case;
but in a non-offloaded cases it could be millions pps.
And in case of sampling over many ethx devices, you can redirect
samples from many other ethx devices.
So making dummy0 the sflow device is a win.
2) Encaping an IFE header implies a much more efficient bpf filter
(IFE ethertype is an excellent discriminator for bpf).
Additional benefit is as mentioned before - redirecting to a device
means you can send it remotely over ethernet to a more powerful
machine without having to cross kernel-userspace. Redirecting instead
of mirroring to tuntap is also an interesting option.
sure, this seems like a good option to have.
generally you have one instance of the sampling agent on a hyper visor or switch.
But, if you have use-cases where monitoring agents run external, sure.
would have preferred if it was optional or an addon and not the default.
Regarding the device, yeah, agree there are pros and cons.
An additional device just to sample packets seems like an overkill.
But, if there is no other other option, and there are benefits to it, no objections.
Hopefully we can add another option on the existing api to skip the device in the
future.
quoted
On 16-10-15 12:34 PM, Roopa Prabhu wrote:
quoted
On 10/12/16, 5:41 AM, Jiri Pirko wrote:
quoted
From: Yotam Gigi <yotam.gi@gmail.com>
quoted
quoted
+
+struct sample_packet_metadata {
+ int sample_size;
+ int orig_size;
+ int ifindex;
+};
+
This metadata does not look extensible.. can it be made to ?
Sure it can...
I will update the userspace API to be more generice: I will drop this struct and
let the user (iproute2 currently) build the netlink packet himself (as Or
Gerlitz suggested).
quoted
quoted
With sflow in context, you need a pair of ifindex numbers to encode ingress and
egress ports.
quoted
What is the use case for both?
I have heard that most monitoring tools have moved to ingress only sampling
because of operational
complexity (use case is sflow). I think hardware also supports ingress and egress
only sampling.
better to have an option to reflect that in the api.
Agree. I will add both ingress and egress ports in the IFE. Both the hardware
Implementation and kernel implementation don't support setting both, but
It is good to have that option.
quoted
quoted
Ideally you would also include a sequence number and a count of the total
number of packets
quoted
quoted
that were candidates for sampling.
Sequence number may make sense (they will help show a gap if something
gets dropped). But i am not sure about the stats consuming such space.
Stats are something that can be queried (tc stats should have a record
of how many bytes/packets )
sure, thats fine.
Will add sequence number.
quoted
quoted
The OVS implementation is a good example, the metadata includes all the
actions applied
quoted
quoted
to the packet in the kernel data path.
Again not sure what the use case would be (and why waste such space
especially when you are sending over the wire with such details).
The sampling function isn't random
if (++s->packet_counter % s->rate == 0) {
This is unsuitable for sFlow, which is specific about the random sampling
function required.
quoted
quoted
BPF, OVS, and the
ULOG statistics module include efficient kernel based random sampling
functions that could be used instead.
quoted
quoted
If i understood correctly, the above is a fallback sampling algorithm.
In the case of the spectrum it already does the sampling in the ASIC
so there is no need to repeat it in software.
Agreed that in that case the sampling approach is not sufficiently
random.
yes. and since the same sampling api will be used for offloaded and non-offloaded
case,
the sampling algo here for the non-offloaded case...can do better .. atleast match
the existing
api efficiency. We would want people to use the same api for the offload and non-
offloaded case.
Yep, spectrum does not have this functionality, and this is why I did not implement in
sample sw implementation too. I agree that it should be there.
In my opinion, it should be optional param (like random_type or something), that the
spectrum offload support will not implement (until hardware will support it).
-----Original Message-----
From: Or Gerlitz [mailto:gerlitz.or@gmail.com]
Sent: Sunday, October 16, 2016 1:27 PM
To: Jiri Pirko <jiri@resnulli.us>
Cc: Linux Netdev List <redacted>; David Miller
[off-list ref]; Yotam Gigi [off-list ref]; Ido Schimmel
[off-list ref]; Elad Raz [off-list ref]; Nogah Frankel
[off-list ref]; Or Gerlitz [off-list ref]; Jamal Hadi Salim
[off-list ref]; geert+renesas@glider.be; Stephen Hemminger
[off-list ref]; Cong Wang [off-list ref];
Guenter Roeck [off-list ref]
Subject: Re: [patch net-next RFC 4/6] Introduce sample tc action
On Wed, Oct 12, 2016 at 3:41 PM, Jiri Pirko [off-list ref] wrote:
quoted
From: Yotam Gigi <yotam.gi@gmail.com>
This action allow the user to sample traffic matched by tc classifier.
The sampling consists of choosing packets randomly, truncating them,
adding some informative metadata regarding the interface and the original
packet size and mark them with specific mark, to allow further tc rules to
match and process. The marked sample packets are then injected into the
device ingress qdisc using netif_receive_skb.
The packets metadata is packed using the ife encapsulation protocol, and
the outer packet's ethernet dest, source and eth_type, along with the
rate, mark and the optional truncation size can be configured from
userspace.
Example:
To sample ingress traffic from interface eth1, and redirect the sampled
the sampled packets to interface dummy0, one may use the commands:
tc qdisc add dev eth1 handle ffff: ingress
tc filter add dev eth1 parent ffff: \
matchall action sample rate 12 mark 17
tc filter add parent ffff: dev eth1 protocol all \
u32 match mark 172 0xff
action mirred egress redirect dev dummy0
Where the first command adds an ingress qdisc and the second starts
sampling every 12'th packet on dev eth0 and marks the sampled packets with
17. The command third catches the sampled packets, which are marked with
17, and redirects them to dev dummy0.
eth0 --> eth1
command third --> third command
Missed that. Thanks :)
don't we need a re-classify directive for the u32 filter to apply
after the marking done by the matchall rule + sample action
or is that implicit?
No, as the packets are re-injected to the ingress qdisc (as described in the
commit message). Reclassify won't work as the sampled packets, which are a copy
of the chosen packets are generated inside the sample action and are not part of
the device packet stream.
@@ -0,0 +1,31 @@+#ifndef __LINUX_TC_SAMPLE_H+#define __LINUX_TC_SAMPLE_H++#include<linux/types.h>+#include<linux/pkt_cls.h>+#include<linux/if_ether.h>++#define TCA_ACT_SAMPLE 26++structtc_sample{+tc_gen;+__u32rate;/* sample rate */+__u32mark;/* mark to put on the sampled packets */+booltruncate;/* whether to truncate the packets */+__u32trunc_size;/* truncation size */+__u8eth_dst[ETH_ALEN];/* encapsulated mac destination */+__u8eth_src[ETH_ALEN];/* encapsulated mac source */+booleth_type_set;/* whether to overrid ethtype */+__u16eth_type;/* encapsulated mac ethtype */+};
overrid --> override
Fixed. Thanks :)
what do you mean by override here, to encapsulate?
No. It’s the IFE header eth_type.
consider using 0 as special value, e.g no truncation and no encapsulation
best if you just define the netlink attributes (document on the RHS
the type, see the uapi
for the new tunnel key action) and let the tc action in-kernel code to
decode them directly
into the non UAPI structure. This way you are extendable and also
avoid having two
structs which is sort of confusing.
I guess that making the API extensible solves this, isn't it?
yes, that might help...
Just wanted to bring up the question/clarification on using mark again
tc qdisc add dev eth1 handle ffff: ingress
tc filter add dev eth1 parent ffff: \
matchall action sample rate 12 mark 17
tc filter add parent ffff: dev eth1 protocol all \
u32 match mark 172 0xff
action mirred egress redirect dev dummy0
Like we discussed @ netdev, mark can be used by other things in the system.
A request to sample on an interface cannot be disruptive.
Does this require mark to be not used elsewhere in the system when sampling is enabled on an interface ?
I guess that making the API extensible solves this, isn't it?
yes, that might help...
Just wanted to bring up the question/clarification on using mark again
tc qdisc add dev eth1 handle ffff: ingress
tc filter add dev eth1 parent ffff: \
matchall action sample rate 12 mark 17
tc filter add parent ffff: dev eth1 protocol all \
u32 match mark 172 0xff
action mirred egress redirect dev dummy0
Like we discussed @ netdev, mark can be used by other things in the system.
A request to sample on an interface cannot be disruptive.
Does this require mark to be not used elsewhere in the system when sampling is
enabled on an interface ?
I think the we can spare the usage of mark, or at least make it optional, as the user
can match on the packets according to the eth_type (as part of the IFE, the user
can set the sampled packet eth_type).
I will do that, and update the documentation as well.