Skip to content

Commit 85ff17d

Browse files
Tyler Liprsunny
Tyler Li
authored and
prsunny
committed
[VRF]: submit vrf feature (sonic-net#943)
Dataplane vrf feature
1 parent 5604566 commit 85ff17d

27 files changed

+3695
-668
lines changed

‎cfgmgr/intfmgr.cpp

+130-31
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ using namespace swss;
1515
#define LAG_PREFIX "PortChannel"
1616
#define LOOPBACK_PREFIX "Loopback"
1717
#define VNET_PREFIX "Vnet"
18+
#define VRF_PREFIX "Vrf"
19+
20+
#define LOOPBACK_DEFAULT_MTU_STR "65536"
1821

1922
IntfMgr::IntfMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, const vector<string> &tableNames) :
2023
Orch(cfgDb, tableNames),
@@ -58,7 +61,7 @@ void IntfMgr::setIntfIp(const string &alias, const string &opCmd,
5861
}
5962
}
6063

61-
void IntfMgr::setIntfVrf(const string &alias, const string vrfName)
64+
void IntfMgr::setIntfVrf(const string &alias, const string &vrfName)
6265
{
6366
stringstream cmd;
6467
string res;
@@ -71,7 +74,97 @@ void IntfMgr::setIntfVrf(const string &alias, const string vrfName)
7174
{
7275
cmd << IP_CMD << " link set " << shellquote(alias) << " nomaster";
7376
}
74-
EXEC_WITH_ERROR_THROW(cmd.str(), res);
77+
int ret = swss::exec(cmd.str(), res);
78+
if (ret)
79+
{
80+
SWSS_LOG_ERROR("Command '%s' failed with rc %d", cmd.str().c_str(), ret);
81+
}
82+
}
83+
84+
void IntfMgr::addLoopbackIntf(const string &alias)
85+
{
86+
stringstream cmd;
87+
string res;
88+
89+
cmd << IP_CMD << " link add " << alias << " mtu " << LOOPBACK_DEFAULT_MTU_STR << " type dummy && ";
90+
cmd << IP_CMD << " link set " << alias << " up";
91+
int ret = swss::exec(cmd.str(), res);
92+
if (ret)
93+
{
94+
SWSS_LOG_ERROR("Command '%s' failed with rc %d", cmd.str().c_str(), ret);
95+
}
96+
}
97+
98+
void IntfMgr::delLoopbackIntf(const string &alias)
99+
{
100+
stringstream cmd;
101+
string res;
102+
103+
cmd << IP_CMD << " link del " << alias;
104+
int ret = swss::exec(cmd.str(), res);
105+
if (ret)
106+
{
107+
SWSS_LOG_ERROR("Command '%s' failed with rc %d", cmd.str().c_str(), ret);
108+
}
109+
}
110+
111+
int IntfMgr::getIntfIpCount(const string &alias)
112+
{
113+
stringstream cmd;
114+
string res;
115+
116+
/* query ip address of the device with master name, it is much faster */
117+
// ip address show {{intf_name}}
118+
// $(ip link show {{intf_name}} | grep -o 'master [^\\s]*') ==> [master {{vrf_name}}]
119+
// | grep inet | grep -v 'inet6 fe80:' | wc -l
120+
cmd << IP_CMD << " address show " << alias
121+
<< " $(" << IP_CMD << " link show " << alias << " | grep -o 'master [^\\s]*')"
122+
<< " | grep inet | grep -v 'inet6 fe80:' | wc -l";
123+
124+
int ret = swss::exec(cmd.str(), res);
125+
if (ret)
126+
{
127+
SWSS_LOG_ERROR("Command '%s' failed with rc %d", cmd.str().c_str(), ret);
128+
return 0;
129+
}
130+
131+
return std::stoi(res);
132+
}
133+
134+
bool IntfMgr::isIntfCreated(const string &alias)
135+
{
136+
vector<FieldValueTuple> temp;
137+
138+
if (m_stateIntfTable.get(alias, temp))
139+
{
140+
SWSS_LOG_DEBUG("Intf %s is ready", alias.c_str());
141+
return true;
142+
}
143+
144+
return false;
145+
}
146+
147+
bool IntfMgr::isIntfChangeVrf(const string &alias, const string &vrfName)
148+
{
149+
vector<FieldValueTuple> temp;
150+
151+
if (m_stateIntfTable.get(alias, temp))
152+
{
153+
for (auto idx : temp)
154+
{
155+
const auto &field = fvField(idx);
156+
const auto &value = fvValue(idx);
157+
if (field == "vrf")
158+
{
159+
if (value == vrfName)
160+
return false;
161+
else
162+
return true;
163+
}
164+
}
165+
}
166+
167+
return false;
75168
}
76169

77170
bool IntfMgr::isIntfStateOk(const string &alias)
@@ -102,6 +195,14 @@ bool IntfMgr::isIntfStateOk(const string &alias)
102195
return true;
103196
}
104197
}
198+
else if (!alias.compare(0, strlen(VRF_PREFIX), VRF_PREFIX))
199+
{
200+
if (m_stateVrfTable.get(alias, temp))
201+
{
202+
SWSS_LOG_DEBUG("Vrf %s is ready", alias.c_str());
203+
return true;
204+
}
205+
}
105206
else if (m_statePortTable.get(alias, temp))
106207
{
107208
SWSS_LOG_DEBUG("Port %s is ready", alias.c_str());
@@ -149,32 +250,38 @@ bool IntfMgr::doIntfGeneralTask(const vector<string>& keys,
149250
return false;
150251
}
151252

152-
// Set Interface VRF except for lo
153-
if (!is_lo)
253+
/* if to change vrf then skip */
254+
if (isIntfChangeVrf(alias, vrf_name))
154255
{
155-
if (!vrf_name.empty())
156-
{
157-
setIntfVrf(alias, vrf_name);
158-
}
159-
m_appIntfTableProducer.set(alias, data);
256+
SWSS_LOG_ERROR("%s can not change to %s directly, skipping", alias.c_str(), vrf_name.c_str());
257+
return true;
160258
}
161-
else
259+
if (is_lo)
162260
{
163-
m_appIntfTableProducer.set("lo", data);
261+
addLoopbackIntf(alias);
164262
}
263+
if (!vrf_name.empty())
264+
{
265+
setIntfVrf(alias, vrf_name);
266+
}
267+
m_appIntfTableProducer.set(alias, data);
268+
m_stateIntfTable.hset(alias, "vrf", vrf_name);
165269
}
166270
else if (op == DEL_COMMAND)
167271
{
168-
// Set Interface VRF except for lo
169-
if (!is_lo)
272+
/* make sure all ip addresses associated with interface are removed, otherwise these ip address would
273+
be set with global vrf and it may cause ip address confliction. */
274+
if (getIntfIpCount(alias))
170275
{
171-
setIntfVrf(alias, "");
172-
m_appIntfTableProducer.del(alias);
276+
return false;
173277
}
174-
else
278+
setIntfVrf(alias, "");
279+
if (is_lo)
175280
{
176-
m_appIntfTableProducer.del("lo");
281+
delLoopbackIntf(alias);
177282
}
283+
m_appIntfTableProducer.del(alias);
284+
m_stateIntfTable.del(alias);
178285
}
179286
else
180287
{
@@ -192,26 +299,21 @@ bool IntfMgr::doIntfAddrTask(const vector<string>& keys,
192299

193300
string alias(keys[0]);
194301
IpPrefix ip_prefix(keys[1]);
195-
bool is_lo = !alias.compare(0, strlen(LOOPBACK_PREFIX), LOOPBACK_PREFIX);
196-
string appKey = (is_lo ? "lo" : keys[0]) + ":" + keys[1];
302+
string appKey = keys[0] + ":" + keys[1];
197303

198304
if (op == SET_COMMAND)
199305
{
200306
/*
201-
* Don't proceed if port/LAG/VLAN is not ready yet.
307+
* Don't proceed if port/LAG/VLAN and intfGeneral are not ready yet.
202308
* The pending task will be checked periodically and retried.
203309
*/
204-
if (!isIntfStateOk(alias))
310+
if (!isIntfStateOk(alias) || !isIntfCreated(alias))
205311
{
206312
SWSS_LOG_DEBUG("Interface is not ready, skipping %s", alias.c_str());
207313
return false;
208314
}
209315

210-
// Set Interface IP except for lo
211-
if (!is_lo)
212-
{
213-
setIntfIp(alias, "add", ip_prefix);
214-
}
316+
setIntfIp(alias, "add", ip_prefix);
215317

216318
std::vector<FieldValueTuple> fvVector;
217319
FieldValueTuple f("family", ip_prefix.isV4() ? IPV4_NAME : IPV6_NAME);
@@ -224,11 +326,8 @@ bool IntfMgr::doIntfAddrTask(const vector<string>& keys,
224326
}
225327
else if (op == DEL_COMMAND)
226328
{
227-
// Set Interface IP except for lo
228-
if (!is_lo)
229-
{
230-
setIntfIp(alias, "del", ip_prefix);
231-
}
329+
setIntfIp(alias, "del", ip_prefix);
330+
232331
m_appIntfTableProducer.del(appKey);
233332
m_stateIntfTable.del(keys[0] + state_db_key_delimiter + keys[1]);
234333
}

‎cfgmgr/intfmgr.h

+6-1
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,16 @@ class IntfMgr : public Orch
2222
Table m_statePortTable, m_stateLagTable, m_stateVlanTable, m_stateVrfTable, m_stateIntfTable;
2323

2424
void setIntfIp(const std::string &alias, const std::string &opCmd, const IpPrefix &ipPrefix);
25-
void setIntfVrf(const std::string &alias, const std::string vrfName);
25+
void setIntfVrf(const std::string &alias, const std::string &vrfName);
2626
bool doIntfGeneralTask(const std::vector<std::string>& keys, const std::vector<FieldValueTuple>& data, const std::string& op);
2727
bool doIntfAddrTask(const std::vector<std::string>& keys, const std::vector<FieldValueTuple>& data, const std::string& op);
2828
void doTask(Consumer &consumer);
2929
bool isIntfStateOk(const std::string &alias);
30+
bool isIntfCreated(const std::string &alias);
31+
bool isIntfChangeVrf(const std::string &alias, const std::string &vrfName);
32+
int getIntfIpCount(const std::string &alias);
33+
void addLoopbackIntf(const std::string &alias);
34+
void delLoopbackIntf(const std::string &alias);
3035
};
3136

3237
}

‎cfgmgr/nbrmgr.cpp

+2-21
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313

1414
using namespace swss;
1515

16-
#define VLAN_PREFIX "Vlan"
17-
#define LAG_PREFIX "PortChannel"
18-
1916
static bool send_message(struct nl_sock *sk, struct nl_msg *msg)
2017
{
2118
bool rc = false;
@@ -67,25 +64,9 @@ bool NbrMgr::isIntfStateOk(const string &alias)
6764
{
6865
vector<FieldValueTuple> temp;
6966

70-
if (!alias.compare(0, strlen(VLAN_PREFIX), VLAN_PREFIX))
71-
{
72-
if (m_stateVlanTable.get(alias, temp))
73-
{
74-
SWSS_LOG_DEBUG("Vlan %s is ready", alias.c_str());
75-
return true;
76-
}
77-
}
78-
else if (!alias.compare(0, strlen(LAG_PREFIX), LAG_PREFIX))
79-
{
80-
if (m_stateLagTable.get(alias, temp))
81-
{
82-
SWSS_LOG_DEBUG("Lag %s is ready", alias.c_str());
83-
return true;
84-
}
85-
}
86-
else if (m_statePortTable.get(alias, temp))
67+
if (m_stateIntfTable.get(alias, temp))
8768
{
88-
SWSS_LOG_DEBUG("Port %s is ready", alias.c_str());
69+
SWSS_LOG_DEBUG("Intf %s is ready", alias.c_str());
8970
return true;
9071
}
9172

‎cfgmgr/vrfmgr.cpp

+59-9
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,16 @@
1010

1111
#define VRF_TABLE_START 1001
1212
#define VRF_TABLE_END 2000
13+
#define TABLE_LOCAL_PREF 1001 // after l3mdev-table
1314

1415
using namespace swss;
1516

1617
VrfMgr::VrfMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, const vector<string> &tableNames) :
1718
Orch(cfgDb, tableNames),
1819
m_appVrfTableProducer(appDb, APP_VRF_TABLE_NAME),
1920
m_appVnetTableProducer(appDb, APP_VNET_TABLE_NAME),
20-
m_stateVrfTable(stateDb, STATE_VRF_TABLE_NAME)
21+
m_stateVrfTable(stateDb, STATE_VRF_TABLE_NAME),
22+
m_stateVrfObjectTable(stateDb, STATE_VRF_OBJECT_TABLE_NAME)
2123
{
2224
for (uint32_t i = VRF_TABLE_START; i < VRF_TABLE_END; i++)
2325
{
@@ -63,6 +65,18 @@ VrfMgr::VrfMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, con
6365
break;
6466
}
6567
}
68+
69+
cmd.str("");
70+
cmd.clear();
71+
cmd << IP_CMD << " rule | grep '^0:'";
72+
if (swss::exec(cmd.str(), res) == 0)
73+
{
74+
cmd.str("");
75+
cmd.clear();
76+
cmd << IP_CMD << " rule add pref " << TABLE_LOCAL_PREF << " table local && " << IP_CMD << " rule del pref 0 && "
77+
<< IP_CMD << " -6 rule add pref " << TABLE_LOCAL_PREF << " table local && " << IP_CMD << " -6 rule del pref 0";
78+
EXEC_WITH_ERROR_THROW(cmd.str(), res);
79+
}
6680
}
6781

6882
uint32_t VrfMgr::getFreeTable(void)
@@ -139,6 +153,19 @@ bool VrfMgr::setLink(const string& vrfName)
139153
return true;
140154
}
141155

156+
bool VrfMgr::isVrfObjExist(const string& vrfName)
157+
{
158+
vector<FieldValueTuple> temp;
159+
160+
if (m_stateVrfObjectTable.get(vrfName, temp))
161+
{
162+
SWSS_LOG_DEBUG("Vrf %s object exist", vrfName.c_str());
163+
return true;
164+
}
165+
166+
return false;
167+
}
168+
142169
void VrfMgr::doTask(Consumer &consumer)
143170
{
144171
SWSS_LOG_ENTER();
@@ -173,20 +200,43 @@ void VrfMgr::doTask(Consumer &consumer)
173200
}
174201
else if (op == DEL_COMMAND)
175202
{
176-
if (!delLink(vrfName))
177-
{
178-
SWSS_LOG_ERROR("Failed to remove vrf netdev %s", vrfName.c_str());
179-
}
180-
181-
m_stateVrfTable.del(vrfName);
182-
203+
/*
204+
* Delay delLink until vrf object deleted in orchagent to ensure fpmsyncd can get vrf ifname.
205+
* Now state VRF_TABLE|Vrf represent vrf exist in appDB, if it exist vrf device is always effective.
206+
* VRFOrch add/del state VRF_OBJECT_TABLE|Vrf to represent object existence. VNETOrch is not do so now.
207+
*/
183208
if (consumer.getTableName() == CFG_VRF_TABLE_NAME)
184209
{
185-
m_appVrfTableProducer.del(vrfName);
210+
vector<FieldValueTuple> temp;
211+
212+
if (m_stateVrfTable.get(vrfName, temp))
213+
{
214+
/* VRFOrch add delay so wait */
215+
if (!isVrfObjExist(vrfName))
216+
{
217+
it++;
218+
continue;
219+
}
220+
221+
m_appVrfTableProducer.del(vrfName);
222+
m_stateVrfTable.del(vrfName);
223+
}
224+
225+
if (isVrfObjExist(vrfName))
226+
{
227+
it++;
228+
continue;
229+
}
186230
}
187231
else
188232
{
189233
m_appVnetTableProducer.del(vrfName);
234+
m_stateVrfTable.del(vrfName);
235+
}
236+
237+
if (!delLink(vrfName))
238+
{
239+
SWSS_LOG_ERROR("Failed to remove vrf netdev %s", vrfName.c_str());
190240
}
191241

192242
SWSS_LOG_NOTICE("Removed vrf netdev %s", vrfName.c_str());

0 commit comments

Comments
 (0)