kpriv/linux-hardened
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

net_sched: add 64bit rate estimators

Browse source 
struct gnet_stats_rate_est contains u32 fields, so the bytes per second
field can wrap at 34360Mbit.

Add a new gnet_stats_rate_est64 structure to get 64bit bps/pps fields,
and switch the kernel to use this structure natively.

This structure is dumped to user space as a new attribute :

TCA_STATS_RATE_EST64

Old tc command will now display the capped bps (to 34360Mbit), instead
of wrapped values, and updated tc command will display correct
information.

Old tc command output, after patch :

eric:~# tc -s -d qd sh dev lo
qdisc pfifo 8001: root refcnt 2 limit 1000p
 Sent 80868245400 bytes 1978837 pkt (dropped 0, overlimits 0 requeues 0)
 rate 34360Mbit 189696pps backlog 0b 0p requeues 0

This patch carefully reorganizes "struct Qdisc" layout to get optimal
performance on SMP.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2013-06-06 08:43:22 -07:00 committed by David S. Miller
parent b41abb42bf
commit 45203a3b38
13 changed files with 54 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/net/act_api.h
View file

@ -18,7 +18,7 @@ struct tcf_common {
struct tcf_t tcfc_tm;
struct gnet_stats_basic_packed tcfc_bstats;
struct gnet_stats_queue tcfc_qstats;
struct gnet_stats_rate_est tcfc_rate_est;
struct gnet_stats_rate_est64 tcfc_rate_est;
spinlock_t tcfc_lock;
struct rcu_head tcfc_rcu;
};

10
include/net/gen_stats.h
View file

@ -30,7 +30,7 @@ extern int gnet_stats_copy_basic(struct gnet_dump *d,
struct gnet_stats_basic_packed *b);
extern int gnet_stats_copy_rate_est(struct gnet_dump *d,
const struct gnet_stats_basic_packed *b,
struct gnet_stats_rate_est *r);
struct gnet_stats_rate_est64 *r);
extern int gnet_stats_copy_queue(struct gnet_dump *d,
struct gnet_stats_queue *q);
extern int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
@ -38,13 +38,13 @@ extern int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
extern int gnet_stats_finish_copy(struct gnet_dump *d);
extern int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est,
struct gnet_stats_rate_est64 *rate_est,
spinlock_t *stats_lock, struct nlattr *opt);
extern void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est);
struct gnet_stats_rate_est64 *rate_est);
extern int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est,
struct gnet_stats_rate_est64 *rate_est,
spinlock_t *stats_lock, struct nlattr *opt);
extern bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
const struct gnet_stats_rate_est *rate_est);
const struct gnet_stats_rate_est64 *rate_est);
#endif

2
include/net/netfilter/xt_rateest.h
View file

@ -6,7 +6,7 @@ struct xt_rateest {
struct gnet_stats_basic_packed bstats;
spinlock_t lock;
/* keep rstats and lock on same cache line to speedup xt_rateest_mt() */
struct gnet_stats_rate_est rstats;
struct gnet_stats_rate_est64 rstats;
/* following fields not accessed in hot path */
struct hlist_node list;

13
include/net/sch_generic.h
View file

@ -58,14 +58,12 @@ struct Qdisc {
* multiqueue device.
*/
#define TCQ_F_WARN_NONWC (1 << 16)
int padded;
u32 limit;
const struct Qdisc_ops *ops;
struct qdisc_size_table __rcu *stab;
struct list_head list;
u32 handle;
u32 parent;
atomic_t refcnt;
struct gnet_stats_rate_est rate_est;
int (*reshape_fail)(struct sk_buff *skb,
struct Qdisc *q);
@ -76,8 +74,9 @@ struct Qdisc {
*/
struct Qdisc *__parent;
struct netdev_queue *dev_queue;
struct Qdisc *next_sched;
struct gnet_stats_rate_est64 rate_est;
struct Qdisc *next_sched;
struct sk_buff *gso_skb;
/*
* For performance sake on SMP, we put highly modified fields at the end
@ -88,8 +87,10 @@ struct Qdisc {
unsigned int __state;
struct gnet_stats_queue qstats;
struct rcu_head rcu_head;
spinlock_t busylock;
u32 limit;
int padded;
atomic_t refcnt;
spinlock_t busylock ____cacheline_aligned_in_smp;
};
static inline bool qdisc_is_running(const struct Qdisc *qdisc)

11
include/uapi/linux/gen_stats.h
View file

@ -9,6 +9,7 @@ enum {
TCA_STATS_RATE_EST,
TCA_STATS_QUEUE,
TCA_STATS_APP,
TCA_STATS_RATE_EST64,
__TCA_STATS_MAX,
};
#define TCA_STATS_MAX (__TCA_STATS_MAX - 1)
@ -37,6 +38,16 @@ struct gnet_stats_rate_est {
__u32 pps;
};
/**
* struct gnet_stats_rate_est64 - rate estimator
* @bps: current byte rate
* @pps: current packet rate
*/
struct gnet_stats_rate_est64 {
__u64 bps;
__u64 pps;
};
/**
* struct gnet_stats_queue - queuing statistics
* @qlen: queue length

12
net/core/gen_estimator.c
View file

@ -82,7 +82,7 @@ struct gen_estimator
{
struct list_head list;
struct gnet_stats_basic_packed *bstats;
struct gnet_stats_rate_est *rate_est;
struct gnet_stats_rate_est64 *rate_est;
spinlock_t *stats_lock;
int ewma_log;
u64 last_bytes;
@ -167,7 +167,7 @@ static void gen_add_node(struct gen_estimator *est)
static
struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats,
const struct gnet_stats_rate_est *rate_est)
const struct gnet_stats_rate_est64 *rate_est)
{
struct rb_node *p = est_root.rb_node;
@ -203,7 +203,7 @@ struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats
*
*/
int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est,
struct gnet_stats_rate_est64 *rate_est,
spinlock_t *stats_lock,
struct nlattr *opt)
{
@ -258,7 +258,7 @@ EXPORT_SYMBOL(gen_new_estimator);
* Note : Caller should respect an RCU grace period before freeing stats_lock
*/
void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est)
struct gnet_stats_rate_est64 *rate_est)
{
struct gen_estimator *e;
@ -290,7 +290,7 @@ EXPORT_SYMBOL(gen_kill_estimator);
* Returns 0 on success or a negative error code.
*/
int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
struct gnet_stats_rate_est *rate_est,
struct gnet_stats_rate_est64 *rate_est,
spinlock_t *stats_lock, struct nlattr *opt)
{
gen_kill_estimator(bstats, rate_est);
@ -306,7 +306,7 @@ EXPORT_SYMBOL(gen_replace_estimator);
* Returns true if estimator is active, and false if not.
*/
bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
const struct gnet_stats_rate_est *rate_est)
const struct gnet_stats_rate_est64 *rate_est)
{
bool res;

22
net/core/gen_stats.c
View file

@ -143,18 +143,30 @@ EXPORT_SYMBOL(gnet_stats_copy_basic);
int
gnet_stats_copy_rate_est(struct gnet_dump *d,
const struct gnet_stats_basic_packed *b,
struct gnet_stats_rate_est *r)
struct gnet_stats_rate_est64 *r)
{
struct gnet_stats_rate_est est;
int res;
if (b && !gen_estimator_active(b, r))
return 0;
est.bps = min_t(u64, UINT_MAX, r->bps);
/* we have some time before reaching 2^32 packets per second */
est.pps = r->pps;
if (d->compat_tc_stats) {
d->tc_stats.bps = r->bps;
d->tc_stats.pps = r->pps;
d->tc_stats.bps = est.bps;
d->tc_stats.pps = est.pps;
}
if (d->tail)
return gnet_stats_copy(d, TCA_STATS_RATE_EST, r, sizeof(*r));
if (d->tail) {
res = gnet_stats_copy(d, TCA_STATS_RATE_EST, &est, sizeof(est));
if (res < 0 || est.bps == r->bps)
return res;
/* emit 64bit stats only if needed */
return gnet_stats_copy(d, TCA_STATS_RATE_EST64, r, sizeof(*r));
}
return 0;
}

2
net/netfilter/xt_rateest.c
View file

@ -18,7 +18,7 @@ static bool
xt_rateest_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
const struct xt_rateest_match_info *info = par->matchinfo;
struct gnet_stats_rate_est *r;
struct gnet_stats_rate_est64 *r;
u_int32_t bps1, bps2, pps1, pps2;
bool ret = true;

2
net/sched/sch_cbq.c
View file

@ -130,7 +130,7 @@ struct cbq_class {
psched_time_t penalized;
struct gnet_stats_basic_packed bstats;
struct gnet_stats_queue qstats;
struct gnet_stats_rate_est rate_est;
struct gnet_stats_rate_est64 rate_est;
struct tc_cbq_xstats xstats;
struct tcf_proto *filter_list;

2
net/sched/sch_drr.c
View file

@ -25,7 +25,7 @@ struct drr_class {
struct gnet_stats_basic_packed bstats;
struct gnet_stats_queue qstats;
struct gnet_stats_rate_est rate_est;
struct gnet_stats_rate_est64 rate_est;
struct list_head alist;
struct Qdisc *qdisc;

2
net/sched/sch_hfsc.c
View file

@ -114,7 +114,7 @@ struct hfsc_class {
struct gnet_stats_basic_packed bstats;
struct gnet_stats_queue qstats;
struct gnet_stats_rate_est rate_est;
struct gnet_stats_rate_est64 rate_est;
unsigned int level; /* class level in hierarchy */
struct tcf_proto *filter_list; /* filter list */
unsigned int filter_cnt; /* filter count */

2
net/sched/sch_htb.c
View file

@ -78,7 +78,7 @@ struct htb_class {
/* general class parameters */
struct gnet_stats_basic_packed bstats;
struct gnet_stats_queue qstats;
struct gnet_stats_rate_est rate_est;
struct gnet_stats_rate_est64 rate_est;
struct tc_htb_xstats xstats; /* our special stats */
int refcnt; /* usage count of this class */

2
net/sched/sch_qfq.c
View file

@ -138,7 +138,7 @@ struct qfq_class {
struct gnet_stats_basic_packed bstats;
struct gnet_stats_queue qstats;
struct gnet_stats_rate_est rate_est;
struct gnet_stats_rate_est64 rate_est;
struct Qdisc *qdisc;
struct list_head alist; /* Link for active-classes list. */
struct qfq_aggregate *agg; /* Parent aggregate. */