PCEtLXtzdWJ0ZW1wbGF0ZSBjb21tb24vaGVhZGVyX2NvbW1vbn0tLT4JPCEtLXtpZiAkX0dFVFsnbW9kJ10gPT0gJ3ZpZXd0aHJlYWQnICYmICFlbXB0eSgkX0dbJ3RpZCddKX0tLT4KCTxsaW5rIHJlbD0iY2Fub25pY2FsIiBocmVmPSJodHRwczovL3d3dy5xaWRhbzEyMy5jb20vYmJzL3RocmVhZC17JF9HWyd0aWQnXX0tMS0xLmh0bWwiIC8+Cgk8IS0te2Vsc2VpZiAkX0dFVFsnbW9kJ10gPT0gJ2ZvcnVtZGlzcGxheScgJiYgIWVtcHR5KCRfR1snZmlkJ10pfS0tPgoJPGxpbmsgcmVsPSJjYW5vbmljYWwiIGhyZWY9Imh0dHBzOi8vd3d3LnFpZGFvMTIzLmNvbS9iYnMvZm9ydW0teyRfR1snZmlkJ119LTEuaHRtbCIgLz4KCTwhLS17ZWxzZWlmICRfR0VUWydtb2QnXSA9PSAnZ3VpZGUnfS0tPgoJPGxpbmsgcmVsPSJjYW5vbmljYWwiIGhyZWY9Imh0dHBzOi8vd3d3LnFpZGFvMTIzLmNvbS9iYnMvZ3VpZGUvIiAvPgoJPCEtLXtlbHNlaWYgJF9HWydiYXNlc2NyaXB0J10gPT0gJ2ZvcnVtJyAmJiAoZW1wdHkoJF9HRVRbJ21vZCddKSB8fCAkX0dFVFsnbW9kJ10gPT0gJ2luZGV4Jyl9LS0+Cgk8bGluayByZWw9ImNhbm9uaWNhbCIgaHJlZj0iaHR0cHM6Ly93d3cucWlkYW8xMjMuY29tL2Jicy8iIC8+Cgk8IS0tey9pZn0tLT4KCXJgbmB0PG1ldGEgbmFtZT0iYXBwbGljYXRpb24tbmFtZSIgY29udGVudD0iJF9HWydzZXR0aW5nJ11bJ2JibmFtZSddIiAvPgoJPG1ldGEgbmFtZT0ibXNhcHBsaWNhdGlvbi10b29sdGlwIiBjb250ZW50PSIkX0dbJ3NldHRpbmcnXVsnYmJuYW1lJ10iIC8+Cgk8IS0te2lmICRfR1snc2V0dGluZyddWydwb3J0YWxzdGF0dXMnXX0tLT48bWV0YSBuYW1lPSJtc2FwcGxpY2F0aW9uLXRhc2siIGNvbnRlbnQ9Im5hbWU9JF9HWydzZXR0aW5nJ11bJ25hdnMnXVsxXVsnbmF2bmFtZSddO2FjdGlvbi11cmk9e2VjaG8gIWVtcHR5KCRfR1snc2V0dGluZyddWydkb21haW4nXVsnYXBwJ11bJ3BvcnRhbCddKSA/ICRfR1snc2NoZW1lJ10uJzovLycuJF9HWydzZXR0aW5nJ11bJ2RvbWFpbiddWydhcHAnXVsncG9ydGFsJ10gOiAkX0dbc2l0ZXVybF0uJ3BvcnRhbC5waHAnfTtpY29uLXVyaT17JF9HW3NpdGV1cmxdfXtJTUdESVJ9L3BvcnRhbC5pY28iIC8+PCEtLXsvaWZ9LS0+Cgk8bWV0YSBuYW1lPSJtc2FwcGxpY2F0aW9uLXRhc2siIGNvbnRlbnQ9Im5hbWU9JF9HWydzZXR0aW5nJ11bJ25hdnMnXVsyXVsnbmF2bmFtZSddO2FjdGlvbi11cmk9e2VjaG8gIWVtcHR5KCRfR1snc2V0dGluZyddWydkb21haW4nXVsnYXBwJ11bJ2ZvcnVtJ10pID8gJF9HWydzY2hlbWUnXS4nOi8vJy4kX0dbJ3NldHRpbmcnXVsnZG9tYWluJ11bJ2FwcCddWydmb3J1bSddIDogJF9HW3NpdGV1cmxdLidmb3J1bS5waHAnfTtpY29uLXVyaT17JF9HW3NpdGV1cmxdfXtJTUdESVJ9L2Jicy5pY28iIC8+Cgk8IS0te2lmICRfR1snc2V0dGluZyddWydncm91cHN0YXR1cyddfS0tPjxtZXRhIG5hbWU9Im1zYXBwbGljYXRpb24tdGFzayIgY29udGVudD0ibmFtZT0kX0dbJ3NldHRpbmcnXVsnbmF2cyddWzNdWyduYXZuYW1lJ107YWN0aW9uLXVyaT17ZWNobyAhZW1wdHkoJF9HWydzZXR0aW5nJ11bJ2RvbWFpbiddWydhcHAnXVsnZ3JvdXAnXSkgPyAkX0dbJ3NjaGVtZSddLic6Ly8nLiRfR1snc2V0dGluZyddWydkb21haW4nXVsnYXBwJ11bJ2dyb3VwJ10gOiAkX0dbc2l0ZXVybF0uJ2dyb3VwLnBocCd9O2ljb24tdXJpPXskX0dbc2l0ZXVybF19e0lNR0RJUn0vZ3JvdXAuaWNvIiAvPjwhLS17L2lmfS0tPgoJPCEtLXtpZiBoZWxwZXJfYWNjZXNzOjpjaGVja19tb2R1bGUoJ2ZlZWQnKX0tLT48bWV0YSBuYW1lPSJtc2FwcGxpY2F0aW9uLXRhc2siIGNvbnRlbnQ9Im5hbWU9JF9HWydzZXR0aW5nJ11bJ25hdnMnXVs0XVsnbmF2bmFtZSddO2FjdGlvbi11cmk9e2VjaG8gIWVtcHR5KCRfR1snc2V0dGluZyddWydkb21haW4nXVsnYXBwJ11bJ2hvbWUnXSkgPyAkX0dbJ3NjaGVtZSddLic6Ly8nLiRfR1snc2V0dGluZyddWydkb21haW4nXVsnYXBwJ11bJ2hvbWUnXSA6ICRfR1tzaXRldXJsXS4naG9tZS5waHAnfTtpY29uLXVyaT17JF9HW3NpdGV1cmxdfXtJTUdESVJ9L2hvbWUuaWNvIiAvPjwhLS17L2lmfS0tPgoJPCEtLXtpZiAkX0dbJ2Jhc2VzY3JpcHQnXSA9PSAnZm9ydW0nICYmICRfR1snc2V0dGluZyddWydhcmNoaXZlciddfS0tPgoJCTxsaW5rIHJlbD0iYXJjaGl2ZXMiIHRpdGxlPSIkX0dbJ3NldHRpbmcnXVsnYmJuYW1lJ10iIGhyZWY9InskX0dbc2l0ZXVybF19YXJjaGl2ZXIvIiAvPgoJPCEtLXsvaWZ9LS0+Cgk8IS0te2lmICFlbXB0eSgkcnNzaGVhZCl9LS0+JHJzc2hlYWQ8IS0tey9pZn0tLT4KCTwhLS17aWYgd2lkdGhhdXRvKCl9LS0+CgkJPGxpbmsgcmVsPSJzdHlsZXNoZWV0IiBpZD0iY3NzX3dpZHRoYXV0byIgdHlwZT0idGV4dC9jc3MiIGhyZWY9J3skX0dbJ3NldHRpbmcnXVsnY3NzcGF0aCddfXtTVFlMRUlEfV93aWR0aGF1dG8uY3NzP3tWRVJIQVNIfScgLz4KCQk8c2NyaXB0IHR5cGU9InRleHQvamF2YXNjcmlwdCI+SFRNTE5PREUuY2xhc3NOYW1lICs9ICcgd2lkdGhhdXRvJzwvc2NyaXB0PgoJPCEtLXsvaWZ9LS0+Cgk8IS0te2lmICRfR1snYmFzZXNjcmlwdCddID09ICdmb3J1bScgfHwgJF9HWydiYXNlc2NyaXB0J10gPT0gJ2dyb3VwJ30tLT4KCQk8c2NyaXB0IHR5cGU9InRleHQvamF2YXNjcmlwdCIgc3JjPSJ7JF9HW3NldHRpbmddW2pzcGF0aF19Zm9ydW0uanM/e1ZFUkhBU0h9Ij48L3NjcmlwdD4KCTwhLS17ZWxzZWlmICRfR1snYmFzZXNjcmlwdCddID09ICdob21lJ30tLT4KCQk8c2NyaXB0IHR5cGU9InRleHQvamF2YXNjcmlwdCIgc3JjPSJ7JF9HW3NldHRpbmddW2pzcGF0aF19aG9tZS5qcz97VkVSSEFTSH0iPjwvc2NyaXB0PgoJPCEtLXtlbHNlaWYgJF9HWydiYXNlc2NyaXB0J10gPT0gJ3BvcnRhbCd9LS0+CgkJPHNjcmlwdCB0eXBlPSJ0ZXh0L2phdmFzY3JpcHQiIHNyYz0ieyRfR1tzZXR0aW5nXVtqc3BhdGhdfXBvcnRhbC5qcz97VkVSSEFTSH0iPjwvc2NyaXB0PgoJPCEtLXsvaWZ9LS0+Cgk8IS0te2lmICRfR1snYmFzZXNjcmlwdCddICE9ICdwb3J0YWwnICYmICRfR0VUWydkaXknXSA9PSAneWVzJyAmJiBjaGVja19kaXlfcGVybSgkdG9waWMpfS0tPgoJCTxzY3JpcHQgdHlwZT0idGV4dC9qYXZhc2NyaXB0IiBzcmM9InskX0dbc2V0dGluZ11banNwYXRoXX1wb3J0YWwuanM/e1ZFUkhBU0h9Ij48L3NjcmlwdD4KCTwhLS17L2lmfS0tPgoJPCEtLXtpZiAkX0dFVFsnZGl5J10gPT0gJ3llcycgJiYgY2hlY2tfZGl5X3Blcm0oJHRvcGljKX0tLT4KCQk8bGluayByZWw9InN0eWxlc2hlZXQiIHR5cGU9InRleHQvY3NzIiBpZD0iZGl5X2NvbW1vbiIgaHJlZj0ieyRfR1snc2V0dGluZyddWydjc3NwYXRoJ119e1NUWUxFSUR9X2Nzc19kaXkuY3NzP3tWRVJIQVNIfSIgLz4KCTwhLS17L2lmfS0tPgo8L2hlYWQ+Cgo8Ym9keSBpZD0ibnZfeyRfR1tiYXNlc2NyaXB0XX0iIGNsYXNzPSJwZ197Q1VSTU9EVUxFfXtpZiAkX0dbJ2Jhc2VzY3JpcHQnXSA9PT0gJ3BvcnRhbCcgJiYgQ1VSTU9EVUxFID09PSAnbGlzdCcgJiYgIWVtcHR5KCRjYXQpfSB7JGNhdFsnYm9keWNzcyddfXsvaWZ9IiBvbmtleWRvd249ImlmKGV2ZW50LmtleUNvZGU9PTI3KSByZXR1cm4gZmFsc2U7Ij4KCTxkaXYgaWQ9ImFwcGVuZF9wYXJlbnQiPjwvZGl2PjxkaXYgaWQ9ImFqYXh3YWl0aWQiPjwvZGl2PgoJPCEtLXtpZiAkX0dFVFsnZGl5J10gPT0gJ3llcycgJiYgY2hlY2tfZGl5X3Blcm0oJHRvcGljKX0tLT4KCQk8IS0te3RlbXBsYXRlIGNvbW1vbi9oZWFkZXJfZGl5fS0tPgoJPCEtLXsvaWZ9LS0+Cgk8IS0te2lmIGNoZWNrX2RpeV9wZXJtKCR0b3BpYyl9LS0+CgkJPCEtLXt0ZW1wbGF0ZSBjb21tb24vaGVhZGVyX2RpeW5hdn0tLT4KCTwhLS17L2lmfS0tPgoJPCEtLXtpZiBDVVJNT0RVTEUgPT0gJ3RvcGljJyAmJiAkdG9waWMgJiYgZW1wdHkoJHRvcGljWyd1c2VoZWFkZXInXSkgJiYgY2hlY2tfZGl5X3Blcm0oJHRvcGljKX0tLT4KCQkkZGl5bmF2Cgk8IS0tey9pZn0tLT4KCTwhLS17aWYgZW1wdHkoJHRvcGljKSB8fCAkdG9waWNbJ3VzZWhlYWRlciddfS0tPgoJCTwhLS17aWYgJF9HWydzZXR0aW5nJ11bJ21vYmlsZSddWydhbGxvd21vYmlsZSddICYmICghJF9HWydzZXR0aW5nJ11bJ2NhY2hlaW5kZXhsaWZlJ10gJiYgISRfR1snc2V0dGluZyddWydjYWNoZXRocmVhZG9uJ10gfHwgJF9HWyd1aWQnXSkgJiYgKCRfR0VUWydkaXknXSAhPSAneWVzJyB8fCAhJF9HRVRbJ2luYWpheCddKSAmJiAoJF9HWydtb2JpbGUnXSAhPSAnJyAmJiAkX0dbJ2Nvb2tpZSddWydtb2JpbGUnXSA9PSAnJyAmJiAkX0dFVFsnbW9iaWxlJ10gIT0gJ25vJyl9LS0+CgkJCTxkaXYgY2xhc3M9InhpMSBibSBibV9jIj4KCQkJICAgIHtsYW5nIHlvdXJfbW9iaWxlX2Jyb3dzZXJ9PGEgaHJlZj0ieyRfR1snc2l0ZXVybCddfWZvcnVtLnBocD9tb2JpbGU9eWVzIj57bGFuZyBnb190b19tb2JpbGV9PC9hPiA8c3BhbiBjbGFzcz0ieGcxIj58PC9zcGFuPiA8YSBocmVmPSIkX0dbJ3NldHRpbmcnXVsnbW9iaWxlJ11bJ25vbW9iaWxldXJsJ10iPntsYW5nIHRvX2JlX2NvbnRpbnVlfTwvYT4KCQkJPC9kaXY+CgkJPCEtLXsvaWZ9LS0+CgkJPCEtLXtpZiAhZW1wdHkoJF9HWydzZXR0aW5nJ11bJ3Nob3J0Y3V0J10pICYmICRfR1snbWVtYmVyJ11bY3JlZGl0c10gPj0gJF9HWydzZXR0aW5nJ11bJ3Nob3J0Y3V0J119LS0+CgkJCTxkaXYgaWQ9InNob3J0Y3V0Ij4KCQkJCTxzcGFuPjxhIGhyZWY9ImphdmFzY3JpcHQ6OyIgaWQ9InNob3J0Y3V0Y2xvc2VpZCIgdGl0bGU9IntsYW5nIGNsb3NlfSI+e2xhbmcgY2xvc2V9PC9hPjwvc3Bhbj4KCQkJCXtsYW5nIHNob3J0Y3V0X25vdGljZX0KCQkJCTxhIGhyZWY9ImphdmFzY3JpcHQ6OyIgaWQ9InNob3J0Y3V0dGlwIj57bGFuZyBzaG9ydGN1dF9hZGR9PC9hPgoKCQkJPC9kaXY+CgkJCTxzY3JpcHQgdHlwZT0idGV4dC9qYXZhc2NyaXB0Ij5zZXRUaW1lb3V0KHNldFNob3J0Y3V0LCAyMDAwKTs8L3NjcmlwdD4KCQk8IS0tey9pZn0tLT4KCQk8ZGl2IGlkPSJ0b3B0YiIgY2xhc3M9ImNsIj4KCQkJPCEtLXtob29rL2dsb2JhbF9jcG5hdl90b3B9LS0+CgkJCTxkaXYgY2xhc3M9IndwIj4KCQkJCTxkaXYgY2xhc3M9InoiPgoJCQkJCTwhLS17bG9vcCAkX0dbJ3NldHRpbmcnXVsndG9wbmF2cyddWzBdICRuYXZ9LS0+CgkJCQkJCTwhLS17aWYgaXNfYXJyYXkoJG5hdikgJiYgJG5hdlsnYXZhaWxhYmxlJ10gJiYgKCEkbmF2WydsZXZlbCddIHx8ICgkbmF2WydsZXZlbCddID09IDEgJiYgJF9HWyd1aWQnXSkgfHwgKCRuYXZbJ2xldmVsJ10gPT0gMiAmJiAkX0dbJ2FkbWluaWQnXSA+IDApIHx8ICgkbmF2WydsZXZlbCddID09IDMgJiYgJF9HWydhZG1pbmlkJ10gPT0gMSkpfS0tPiRuYXZbY29kZV08IS0tey9pZn0tLT4KCQkJCQk8IS0tey9sb29wfS0tPgoJCQkJCTwhLS17aG9vay9nbG9iYWxfY3BuYXZfZXh0cmExfS0tPgoJCQkJPC9kaXY+CgkJCQk8ZGl2IGNsYXNzPSJ5Ij4KCQkJCQk8YSBpZD0ic3dpdGNoYmxpbmQiIGhyZWY9ImphdmFzY3JpcHQ6OyIgb25jbGljaz0idG9nZ2xlQmxpbmQodGhpcykiIHRpdGxlPSJ7bGFuZyBzd2l0Y2hfYmxpbmR9IiBjbGFzcz0ic3dpdGNoYmxpbmQiPjwvYT4KCQkJCQk8IS0te2hvb2svZ2xvYmFsX2NwbmF2X2V4dHJhMn0tLT4KCQkJCQk8IS0te2xvb3AgJF9HWydzZXR0aW5nJ11bJ3RvcG5hdnMnXVsxXSAkbmF2fS0tPgoJCQkJCQk8IS0te2lmIGlzX2FycmF5KCRuYXYpICYmICRuYXZbJ2F2YWlsYWJsZSddICYmICghJG5hdlsnbGV2ZWwnXSB8fCAoJG5hdlsnbGV2ZWwnXSA9PSAxICYmICRfR1sndWlkJ10pIHx8ICgkbmF2WydsZXZlbCddID09IDIgJiYgJF9HWydhZG1pbmlkJ10gPiAwKSB8fCAoJG5hdlsnbGV2ZWwnXSA9PSAzICYmICRfR1snYWRtaW5pZCddID09IDEpKX0tLT4kbmF2W2NvZGVdPCEtLXsvaWZ9LS0+CgkJCQkJPCEtLXsvbG9vcH0tLT4KCQkJCQk8IS0te2lmIGVtcHR5KCRfR1snZGlzYWJsZWR3aWR0aGF1dG8nXSkgJiYgJF9HWydzZXR0aW5nJ11bJ3N3aXRjaHdpZHRoYXV0byddfS0tPgoJCQkJCQk8YSBocmVmPSJqYXZhc2NyaXB0OjsiIGlkPSJzd2l0Y2h3aWR0aCIgb25jbGljaz0id2lkdGhhdXRvKHRoaXMpIiB0aXRsZT0ie2lmIHdpZHRoYXV0bygpfXtsYW5nIHN3aXRjaF9uYXJyb3d9e2Vsc2V9e2xhbmcgc3dpdGNoX3dpZGV9ey9pZn0iIGNsYXNzPSJzd2l0Y2h3aWR0aCI+PCEtLXtpZiB3aWR0aGF1dG8oKX0tLT57bGFuZyBzd2l0Y2hfbmFycm93fTwhLS17ZWxzZX0tLT57bGFuZyBzd2l0Y2hfd2lkZX08IS0tey9pZn0tLT48L2E+CgkJCQkJPCEtLXsvaWZ9LS0+CgkJCQkJPCEtLXtpZiAkX0dbJ3VpZCddICYmICFlbXB0eSgkX0dbJ3N0eWxlJ11bJ2V4dHN0eWxlJ10pfS0tPjxhIGlkPSJzc2xjdCIgaHJlZj0iamF2YXNjcmlwdDo7IiBvbm1vdXNlb3Zlcj0iZGVsYXlTaG93KHRoaXMsIGZ1bmN0aW9uKCkge3Nob3dNZW51KHsnY3RybGlkJzonc3NsY3QnLCdwb3MnOiczNCEnfSl9KTsiPntsYW5nIGNoYW5nZXN0eWxlfTwvYT48IS0tey9pZn0tLT4KCQkJCQk8IS0te2lmIGNoZWNrX2RpeV9wZXJtKCR0b3BpYyl9LS0+CgkJCQkJCSRkaXluYXYKCQkJCQk8IS0tey9pZn0tLT4KCQkJCTwvZGl2PgoJCQk8L2Rpdj4KCQk8L2Rpdj4KCgkJPCEtLXtpZiAhSVNfUk9CT1R9LS0+CgkJCTwhLS17aWYgJF9HWyd1aWQnXSAmJiAhJF9HWydzZXR0aW5nJ11bJ2JiY2xvc2VkJ10gJiYgZW1wdHkoJF9HWydtZW1iZXInXVsnZnJlZXplJ10pICYmICRfR1snbWVtYmVyJ11bJ2dyb3VwaWQnXSAhPSA1fS0tPgoJCQk8dWwgaWQ9Im15cHJvbXB0X21lbnUiIGNsYXNzPSJwX3BvcCIgc3R5bGU9ImRpc3BsYXk6IG5vbmU7Ij4KCQkJCTxsaT48YSBocmVmPSJob21lLnBocD9tb2Q9c3BhY2UmZG89cG0iIGlkPSJwbV9udGMiIHN0eWxlPSJiYWNrZ3JvdW5kLXJlcGVhdDogbm8tcmVwZWF0OyBiYWNrZ3JvdW5kLXBvc2l0aW9uOiAwIDUwJTsiPjxlbSBjbGFzcz0icHJvbXB0X25ld3N7aWYgZW1wdHkoJF9HW21lbWJlcl1bbmV3cG1dKX1fMHsvaWZ9Ij48L2VtPntsYW5nIHBtX2NlbnRlcn08L2E+PC9saT4KCQkJCTwhLS17aWYgJF9HWydzZXR0aW5nJ11bJ2ZvbGxvd3N0YXR1cyddfS0tPgoJCQkJCTxsaT48YSBocmVmPSJob21lLnBocD9tb2Q9Zm9sbG93JmRvPWZvbGxvd2VyIj48ZW0gY2xhc3M9InByb21wdF9mb2xsb3dlcntpZiBlbXB0eSgkX0dbbWVtYmVyXVtuZXdwcm9tcHRfbnVtXVtmb2xsb3dlcl0pfV8wey9pZn0iPjwvZW0+PCEtLXtsYW5nIG5vdGljZV9pbnRlcmFjdGl2ZV9mb2xsb3dlcn0tLT57aWYgJF9HW21lbWJlcl1bbmV3cHJvbXB0X251bV1bZm9sbG93ZXJdfSgkX0dbbWVtYmVyXVtuZXdwcm9tcHRfbnVtXVtmb2xsb3dlcl0pey9pZn08L2E+PC9saT4KCQkJCQk8IS0te2lmICRfR1ttZW1iZXJdW25ld3Byb21wdF0gJiYgJF9HW21lbWJlcl1bbmV3cHJvbXB0X251bV1bZm9sbG93XX0tLT4KCQkJCQkJPGxpPjxhIGhyZWY9ImhvbWUucGhwP21vZD1mb2xsb3ciPjxlbSBjbGFzcz0icHJvbXB0X2NvbmNlcm4iPjwvZW0+PCEtLXtsYW5nIG5vdGljZV9pbnRlcmFjdGl2ZV9mb2xsb3d9LS0+KCRfR1ttZW1iZXJdW25ld3Byb21wdF9udW1dW2ZvbGxvd10pPC9hPjwvbGk+CgkJCQkJPCEtLXsvaWZ9LS0+CgkJCQk8IS0tey9pZn0tLT4KCQkJCTwhLS17aWYgJF9HW21lbWJlcl1bbmV3cHJvbXB0XX0tLT4KCQkJCQk8IS0te2xvb3AgJF9HWydtZW1iZXInXVsnY2F0ZWdvcnlfbnVtJ10gJGtleSAkdmFsfS0tPgoJCQkJCQk8bGk+PGEgaHJlZj0iaG9tZS5waHA/bW9kPXNwYWNlJmRvPW5vdGljZSZ2aWV3PSRrZXkiPjxlbSBjbGFzcz0ibm90aWNlXyRrZXkiPjwvZW0+PCEtLXtlY2hvIGxhbmcoJ3RlbXBsYXRlJywgJ25vdGljZV8nLiRrZXkpfS0tPig8c3BhbiBjbGFzcz0icnEiPiR2YWw8L3NwYW4+KTwvYT48L2xpPgoJCQkJCTwhLS17L2xvb3B9LS0+CgkJCQk8IS0tey9pZn0tLT4KCQkJCTwhLS17aWYgZW1wdHkoJF9HWydjb29raWUnXVsnaWdub3JlX25vdGljZSddKX0tLT4KCQkJCQk8bGkgY2xhc3M9Imlnbm9yZV9ub3RpY2VsaSI+PGEgaHJlZj0iamF2YXNjcmlwdDo7IiBvbmNsaWNrPSJzZXRjb29raWUoJ2lnbm9yZV9ub3RpY2UnLCAxKTtoaWRlTWVudSgnbXlwcm9tcHRfbWVudScpIiB0aXRsZT0ie2xhbmcgdGVtcG9yYXJpbHlfdG9fcmVtaW5kfSI+PGVtIGNsYXNzPSJpZ25vcmVfbm90aWNlIj48L2VtPjwvYT48L2xpPgoJCQkJPCEtLXsvaWZ9LS0+CgkJCTwvdWw+CgkJCTwhLS17L2lmfS0tPgoJCQk8IS0te2lmICRfR1sndWlkJ10gJiYgIWVtcHR5KCRfR1snc3R5bGUnXVsnZXh0c3R5bGUnXSl9LS0+CgkJCQk8ZGl2IGlkPSJzc2xjdF9tZW51IiBjbGFzcz0iY2wgcF9wb3AiIHN0eWxlPSJkaXNwbGF5OiBub25lOyI+CgkJCQkJPCEtLXtpZiAhJF9HW3N0eWxlXVtkZWZhdWx0ZXh0c3R5bGVdfS0tPjxzcGFuIGNsYXNzPSJzc2xjdF9idG4iIG9uY2xpY2s9ImV4dHN0eWxlKCcnKSIgdGl0bGU9IntsYW5nIGRlZmF1bHR9Ij48aT48L2k+PC9zcGFuPjwhLS17L2lmfS0tPgoJCQkJCTwhLS17bG9vcCAkX0dbJ3N0eWxlJ11bJ2V4dHN0eWxlJ10gJGV4dHN0eWxlfS0tPgoJCQkJCQk8c3BhbiBjbGFzcz0ic3NsY3RfYnRuIiBvbmNsaWNrPSJleHRzdHlsZSgnJGV4dHN0eWxlWzBdJykiIHRpdGxlPSIkZXh0c3R5bGVbMV0iPjxpIHN0eWxlPSdiYWNrZ3JvdW5kOiRleHRzdHlsZVsyXSc+PC9pPjwvc3Bhbj4KCQkJCQk8IS0tey9sb29wfS0tPgoJCQkJPC9kaXY+CgkJCTwhLS17L2lmfS0tPgoJCQk8IS0te2lmICRfR1sndWlkJ119LS0+CgkJCQk8dWwgaWQ9Im15aXRlbV9tZW51IiBjbGFzcz0icF9wb3AiIHN0eWxlPSJkaXNwbGF5OiBub25lOyI+CgkJCQkJPCEtLXtpZiAkX0dbJ3NldHRpbmcnXVsnZm9ydW1zdGF0dXMnXX0tLT48bGk+PGEgaHJlZj0iaG9tZS5waHA/bW9kPXNwYWNlJmRvPXRocmVhZCZ2aWV3PW1lIj57bGFuZyBteXBvc3R9PC9hPjwvbGk+PCEtLXsvaWZ9LS0+CgkJCQkJPCEtLXtpZiAkX0dbJ3NldHRpbmcnXVsnZmF2b3JpdGVzdGF0dXMnXX0tLT48bGk+PGEgaHJlZj0iaG9tZS5waHA/bW9kPXNwYWNlJmRvPWZhdm9yaXRlJnZpZXc9bWUiPntsYW5nIGZhdm9yaXRlfTwvYT48L2xpPjwhLS17L2lmfS0tPgoJCQkJCTwhLS17aWYgJF9HWydzZXR0aW5nJ11bJ2ZyaWVuZHN0YXR1cyddfS0tPjxsaT48YSBocmVmPSJob21lLnBocD9tb2Q9c3BhY2UmZG89ZnJpZW5kIj57bGFuZyBmcmllbmRzfTwvYT48L2xpPjwhLS17L2lmfS0tPgoJCQkJCTwhLS17aG9vay9nbG9iYWxfbXlpdGVtX2V4dHJhfS0tPgoJCQkJPC91bD4KCQkJPCEtLXsvaWZ9LS0+CgkJCTwhLS17c3VidGVtcGxhdGUgY29tbW9uL2hlYWRlcl9xbWVudX0tLT4KCQk8IS0tey9pZn0tLT4KCgkJPCEtLXthZC9oZWFkZXJiYW5uZXIvd3AgYV9ofS0tPgoJCTxkaXYgaWQ9ImhkIj4KCQkJPGRpdiBjbGFzcz0id3AiPgoJCQkJPGRpdiBjbGFzcz0iaGRjIGNsIj4KCQkJCQk8IS0te2V2YWwgJG1uaWQgPSBnZXRjdXJyZW50bmF2KCk7fS0tPgoJCQkJCTxoMj48IS0te2lmICFpc3NldCgkX0dbJ3NldHRpbmcnXVsnbmF2bG9nb3MnXVskbW5pZF0pfS0tPjxhIGhyZWY9IntpZiAkX0dbJ3NldHRpbmcnXVsnZG9tYWluJ11bJ2FwcCddWydkZWZhdWx0J119eyRfR1snc2NoZW1lJ119Oi8veyRfR1snc2V0dGluZyddWydkb21haW4nXVsnYXBwJ11bJ2RlZmF1bHQnXX0ve2Vsc2V9Li97L2lmfSIgdGl0bGU9IiRfR1snc2V0dGluZyddWydiYm5hbWUnXSI+eyRfR1snc3R5bGUnXVsnYm9hcmRsb2dvJ119PC9hPjwhLS17ZWxzZX0tLT4kX0dbJ3NldHRpbmcnXVsnbmF2bG9nb3MnXVskbW5pZF08IS0tey9pZn0tLT48L2gyPgoJCQkJCTwhLS17dGVtcGxhdGUgY29tbW9uL2hlYWRlcl91c2Vyc3RhdHVzfS0tPgoJCQkJPC9kaXY+CgoJCQkJPGRpdiBpZD0ibnYiPgoJCQkJCTxhIGhyZWY9ImphdmFzY3JpcHQ6OyIgaWQ9InFtZW51IiBvbm1vdXNlb3Zlcj0iZGVsYXlTaG93KHRoaXMsIGZ1bmN0aW9uICgpIHtzaG93TWVudSh7J2N0cmxpZCc6J3FtZW51JywncG9zJzonMzQhJywnY3RybGNsYXNzJzonYScsJ2R1cmF0aW9uJzoyfSk7c2hvd0ZvcnVtbWVudSgkX0dbZmlkXSk7fSkiPntsYW5nIG15X25hdn08L2E+CgkJCQkJPHVsPgoJCQkJCQk8IS0te2xvb3AgJF9HWydzZXR0aW5nJ11bJ25hdnMnXSAkbmF2fS0tPgoJCQkJCQkJPCEtLXtpZiBpc19hcnJheSgkbmF2KSAmJiAkbmF2WydhdmFpbGFibGUnXSAmJiAoISRuYXZbJ2xldmVsJ10gfHwgKCRuYXZbJ2xldmVsJ10gPT0gMSAmJiAkX0dbJ3VpZCddKSB8fCAoJG5hdlsnbGV2ZWwnXSA9PSAyICYmICRfR1snYWRtaW5pZCddID4gMCkgfHwgKCRuYXZbJ2xldmVsJ10gPT0gMyAmJiAkX0dbJ2FkbWluaWQnXSA9PSAxKSl9LS0+PGxpIHtpZiAkbW5pZCA9PSAkbmF2W25hdmlkXSB8fCBzdWJzdHIoJF9TRVJWRVJbJ1JFUVVFU1RfVVJJJ10sIDEpID09IHN0cl9yZXBsYWNlKCcuLycsICcnLCAkbmF2W2ZpbGVuYW1lXSl9Y2xhc3M9ImEiIHsvaWZ9JG5hdltuYXZdPjwvbGk+PCEtLXsvaWZ9LS0+CgkJCQkJCTwhLS17L2xvb3B9LS0+CgkJCQkJPC91bD4KCQkJCQk8IS0te2hvb2svZ2xvYmFsX25hdl9leHRyYX0tLT4KCQkJCTwvZGl2PgoJCQkJPCEtLXtpZiAhZW1wdHkoJF9HWydzZXR0aW5nJ11bJ3BsdWdpbnMnXVsnanNtZW51J10pfS0tPgoJCQkJCTx1bCBjbGFzcz0icF9wb3AgaF9wb3AiIGlkPSJwbHVnaW5fbWVudSIgc3R5bGU9ImRpc3BsYXk6IG5vbmUiPgoJCQkJCTwhLS17bG9vcCAkX0dbJ3NldHRpbmcnXVsncGx1Z2lucyddWydqc21lbnUnXSAkbW9kdWxlfS0tPgoJCQkJCQkgPCEtLXtpZiBpbl9hcnJheSgkbW9kdWxlWydhZG1pbmlkJ10sIGFycmF5KDAsIC0xKSkgfHwgKCRtb2R1bGVbJ2FkbWluaWQnXSAmJiAkX0dbJ2FkbWluaWQnXSA+IDAgJiYgJG1vZHVsZVsnYWRtaW5pZCddID49ICRfR1snYWRtaW5pZCddKX0tLT4KCQkJCQkJIDxsaT4kbW9kdWxlW3VybF08L2xpPgoJCQkJCQkgPCEtLXsvaWZ9LS0+CgkJCQkJPCEtLXsvbG9vcH0tLT4KCQkJCQk8L3VsPgoJCQkJPCEtLXsvaWZ9LS0+CgkJCQkkX0dbc2V0dGluZ11bbWVudW5hdnNdCgkJCQk8ZGl2IGlkPSJtdSIgY2xhc3M9ImNsIj4KCQkJCTwhLS17aWYgJF9HWydzZXR0aW5nJ11bJ3N1Ym5hdnMnXX0tLT4KCQkJCQk8IS0te2xvb3AgJF9HW3NldHRpbmddW3N1Ym5hdnNdICRuYXZpZCAkc3VibmF2fS0tPgoJCQkJCQk8IS0te2lmICRfR1snc2V0dGluZyddWyduYXZzdWJob3ZlciddIHx8ICRtbmlkID09ICRuYXZpZH0tLT4KCQkJCQkJPHVsIGNsYXNzPSJjbCB7aWYgJG1uaWQgPT0gJG5hdmlkfWN1cnJlbnR7L2lmfSIgaWQ9InNuYXZfJG5hdmlkIntpZiAkbW5pZCAhPSAkbmF2aWR9IHN0eWxlPSJkaXNwbGF5Om5vbmUiey9pZn0+CgkJCQkJCSRzdWJuYXYKCQkJCQkJPC91bD4KCQkJCQkJPCEtLXsvaWZ9LS0+CgkJCQkJPCEtLXsvbG9vcH0tLT4KCQkJCTwhLS17L2lmfS0tPgoJCQkJPC9kaXY+CgkJCQk8IS0te2FkL3N1Ym5hdmJhbm5lci9hX211fS0tPgoJCQkJPCEtLXtzdWJ0ZW1wbGF0ZSBjb21tb24vcHVic2VhcmNoZm9ybX0tLT4KCQkJPC9kaXY+CgkJPC9kaXY+CgoJCTwhLS17aG9vay9nbG9iYWxfaGVhZGVyfS0tPgoJPCEtLXsvaWZ9LS0+CgoJPGRpdiBpZD0id3AiIGNsYXNzPSJ3cCI+Cg==
qidao123.com ToB IT社区-企服评测·应用市场»论坛 › 容器及微服务 › 容器及微服务 › 基于 Prometheus+Grafana+Alertmanager 搭建 K8S 云监 ...
返回列表 发新帖

基于 Prometheus+Grafana+Alertmanager 搭建 K8S 云监控告警平台（附配置

[复制链接]
发表于 2024-10-8 02:09:07 | 显示全部楼层 |阅读模式

马上注册，结交更多好友，享用更多功能，让你轻松玩转社区。

您需要 登录 才可以下载或查看，没有账号？立即注册

×
目录

  • 一、机器规划
  • 二、摆设安装 node-exporter、prometheus、Grafana、kube-state-metrics

    • 1、创建 monitor-sa 定名空间
    • 2、安装node-exporter组件

      • 2.1、分析
      • 2.2、应用资源清单
      • 2.3、通过node-exporter采集数据

    • 3、k8s 集群中摆设 prometheus

      • 3.1、创建一个 sa 账号
      • 3.2、将 sa 账号 monitor 通过 clusterrolebing 绑定到 clusterrole 上
      • 3.3、创建数据目录
      • 3.4、安装prometheus

        • 3.4.1、将 prometheus.yml 文件以 ConfigMap 的形式举行管理
        • 3.4.2、应用 cm 资源清单
        • 3.4.3、通过 Deployment 摆设 prometheus
        • 3.4.4、应用 prometheus 资源清单
        • 3.4.5、给 prometheus 的 pod 创建一个 svc
        • 3.4.6、应用 svc 资源清单

      • 3.5、访问prometheus UI界面
      • 3.6、检察配置的服务发现

    • 4、prometheus热更新

      • 4.1、热加载 prometheus
      • 4.2、暴力重启 prometheus

    • 5、Grafana安装和配置

      • 5.1、下载 Grafana 必要的镜像
      • 5.2、在 k8s 集群各个节点导入 Grafana 镜像
      • 5.3、master 节点创建 grafana.yaml
      • 5.4、检察 Grafana 的 pod 和 svc
      • 5.5、检察 Grafana UI 界面
      • 5.6、给 Grafana 接入 Prometheus 数据源
      • 5.7、获取监控监控模板
      • 5.8、导入监控监控模板

    • 6、安装配置 kube-state-metrics 组件

      • 6.1、什么是 kube-state-metrics
      • 6.2、创建 sa ，并举行授权
      • 6.3、创建并应用 kube-state-metrics-deploy.yaml 文件
      • 6.4、创建并应用 kube-state-metrics-svc.yaml 文件
      • 6.5、获取 kube-state-metrics json 文件
      • 6.6、向 Grafana 导入 kube-state-metrics json 文件


  • 三、安装和配置 Alertmanager -- 发送告警到 QQ 邮箱

    • 1、将 alertmanager-cm.yaml 文件以 cm 形式举行管理

      • 1.1、alertmanager配置文件分析

    • 2、重新生成并应用 prometheus-cfg.yaml 文件
    • 3、重新生成 prometheus-deploy.yaml 文件

      • 3.1、创建一个名为 etcd-certs 的 Secret
      • 3.2、应用 prometheus-deploy.yaml 文件

    • 4、重新生成并创建 alertmanager-svc.yaml 文件
    • 5、访问 prometheus UI 界面

      • 5.1、【error】kube-controller-manager、etcd、kube-proxy、kube-scheduler 组件 connection refused

        • 5.1.1、kube-proxy
        • 5.1.2、kube-controller-manager
        • 5.1.3、kube-schedule
        • 5.1.4、etcd


    • 6、点击Alerts，检察
    • 7、把controller-manager的cpu使用率大于90%展开
    • 8、登录 alertmanager UI
    • 9、登录 QQ 邮箱检察告警信息

  • 四、配置 Alertmanager 报警 -- 发送告警到钉钉

    • 1、手机端拉群
    • 2、创建自定义机器人
    • 3、获取钉钉的 Webhook 插件
    • 4、启动钉钉告警插件
    • 5、对 alertmanager-cm.yaml 文件做备份
    • 6、重新生成新的 alertmanager-cm.yaml 文件
    • 7、重建资源以生效
    • 8、结果


一、机器规划

角色主机名ip 地点masterk8s-master1192.168.112.10nodek8s-node1192.168.112.20nodek8s-node2192.168.112.30平台VMware Workstation操纵系统CentOS Linux release 7.9.2009 (Core)内存、CPU4C4G磁盘大小20G SCSI二、摆设安装 node-exporter、prometheus、Grafana、kube-state-metrics

1、创建 monitor-sa 定名空间

master 节点操纵
  1. kubectl create ns monitor-sa
复制代码
2、安装node-exporter组件

master 节点操纵
  1. cat >> node-export.yaml  <<EOF
  2. apiVersion: apps/v1
  3. kind: DaemonSet
  4. metadata:
  5.   name: node-exporter
  6.   namespace: monitor-sa
  7.   labels:
  8.     name: node-exporter
  9. spec:
  10.   selector:
  11.     matchLabels:
  12.      name: node-exporter
  13.   template:
  14.     metadata:
  15.       labels:
  16.         name: node-exporter
  17.     spec:
  18.       hostPID: true
  19.       hostIPC: true
  20.       hostNetwork: true
  21.       containers:
  22.       - name: node-exporter
  23.         image: prom/node-exporter:v0.16.0
  24.         ports:
  25.         - containerPort: 9100
  26.         resources:
  27.           requests:
  28.             cpu: 0.15
  29.         securityContext:
  30.           privileged: true
  31.         args:
  32.         - --path.procfs
  33.         - /host/proc
  34.         - --path.sysfs
  35.         - /host/sys
  36.         - --collector.filesystem.ignored-mount-points
  37.         - '"^/(sys|proc|dev|host|etc)($|/)"'
  38.         volumeMounts:
  39.         - name: dev
  40.           mountPath: /host/dev
  41.         - name: proc
  42.           mountPath: /host/proc
  43.         - name: sys
  44.           mountPath: /host/sys
  45.         - name: rootfs
  46.           mountPath: /rootfs
  47.       tolerations:
  48.       - key: "node-role.kubernetes.io/master"
  49.         operator: "Exists"
  50.         effect: "NoSchedule"
  51.       volumes:
  52.         - name: proc
  53.           hostPath:
  54.             path: /proc
  55.         - name: dev
  56.           hostPath:
  57.             path: /dev
  58.         - name: sys
  59.           hostPath:
  60.             path: /sys
  61.         - name: rootfs
  62.           hostPath:
  63.             path: /
  64. EOF
复制代码
  1. kubectl apply -f node-export.yaml
  3. kubectl get pods -n monitor-sa -l name=node-exporter
复制代码

4.2、暴力重启 prometheus

热加载速度比较慢，可以暴力重启prometheus
如修改上面的prometheus-cfg.yaml文件之后，可执行如下强制删除
  1. # curl http://<master-ip>:9100/metrics
  3. curl http://192.168.112.10:9100/metrics
复制代码
线上最好热加载，暴力删除可能造成监控监控数据的丢失
5、Grafana安装和配置

5.1、下载 Grafana 必要的镜像
  1. kubectl create serviceaccount monitor -n monitor-sa
复制代码
5.2、在 k8s 集群各个节点导入 Grafana 镜像
  1. kubectl create clusterrolebinding monitor-clusterrolebinding -n monitor-sa --clusterrole=cluster-admin  --serviceaccount=monitor-sa:monitor
复制代码



5.3、master 节点创建 grafana.yaml

[code]cat >> grafana.yaml
回复

使用道具 举报

返回列表 发新帖
高级模式
B Color Image Link Quote Code Smilies
您需要登录后才可以回帖 登录 | 立即注册

本版积分规则

登录后关闭弹窗

登录参与点评抽奖  加入IT实名职场社区
去登录

QQ|手机版|qidao123.com IT社区;IT企服评测▪应用市场 ( 浙ICP备20004199|浙ICP备20004199号 )|网站地图

GMT+8, 2026-11-14 09:14 , Processed in 0.037999 second(s), 7 queries , Redis On.

Powered by Discuz! X3.5 Licensed

© 2001-2026 Discuz! Team.

快速回复 返回顶部 返回列表